touch screen input

Signed-off-by: dignow <linlong1265@gmail.com>

flutter/lib/common/widgets/remote_input.dart

@@ -1,7 +1,13 @@
 import 'package:flutter/material.dart';
 import 'package:flutter/services.dart';
+import 'package:flutter/gestures.dart';
 
-import '../../models/input_model.dart';
+import 'package:flutter_hbb/models/platform_model.dart';
+import 'package:flutter_hbb/common.dart';
+import 'package:flutter_hbb/models/model.dart';
+import 'package:flutter_hbb/models/input_model.dart';
+
+import './gestures.dart';
 
 class RawKeyFocusScope extends StatelessWidget {
   final FocusNode? focusNode;
@@ -30,6 +36,290 @@ class RawKeyFocusScope extends StatelessWidget {
   }
 }
 
+class RawTouchGestureDetectorRegion extends StatefulWidget {
+  final Widget child;
+  final FFI ffi;
+
+  late final InputModel inputModel = ffi.inputModel;
+  late final FfiModel ffiModel = ffi.ffiModel;
+
+  RawTouchGestureDetectorRegion({
+    required this.child,
+    required this.ffi,
+  });
+
+  @override
+  State<RawTouchGestureDetectorRegion> createState() =>
+      _RawTouchGestureDetectorRegionState();
+}
+
+  /// touchMode only:
+  ///   LongPress -> right click
+  ///   OneFingerPan -> start/end -> left down start/end
+  ///   onDoubleTapDown -> move to
+  ///   onLongPressDown => move to
+  ///
+  /// mouseMode only:
+  ///   DoubleFiner -> right click
+  ///   HoldDrag -> left drag
+class _RawTouchGestureDetectorRegionState
+    extends State<RawTouchGestureDetectorRegion> {
+  Offset _cacheLongPressPosition = Offset(0, 0);
+  double _mouseScrollIntegral = 0; // mouse scroll speed controller
+  double _scale = 1;
+
+  PointerDeviceKind? lastDeviceKind;
+
+  FFI get ffi => widget.ffi;
+  FfiModel get ffiModel => widget.ffiModel;
+  InputModel get inputModel => widget.inputModel;
+  bool get handleTouch => isDesktop || ffiModel.touchMode;
+  SessionID get sessionId => ffi.sessionId;
+
+  @override
+  Widget build(BuildContext context) {
+    return RawGestureDetector(
+      child: widget.child,
+      gestures: makeGestures(context),
+    );
+  }
+
+  onTapDown(TapDownDetails d) {
+    lastDeviceKind = d.kind;
+    if (lastDeviceKind != PointerDeviceKind.touch) {
+      return;
+    }
+    if (handleTouch) {
+      ffi.cursorModel.move(d.localPosition.dx, d.localPosition.dy);
+    }
+    inputModel.tapDown(MouseButtons.left);
+  }
+
+  onTapUp(TapUpDetails d) {
+    if (lastDeviceKind != PointerDeviceKind.touch) {
+      return;
+    }
+    if (handleTouch) {
+      ffi.cursorModel.move(d.localPosition.dx, d.localPosition.dy);
+    }
+    inputModel.tapUp(MouseButtons.left);
+  }
+
+  onDoubleTapDown(TapDownDetails d) {
+    lastDeviceKind = d.kind;
+    if (lastDeviceKind != PointerDeviceKind.touch) {
+      return;
+    }
+    if (handleTouch) {
+      ffi.cursorModel.move(d.localPosition.dx, d.localPosition.dy);
+    }
+  }
+
+  onDoubleTap() {
+    if (lastDeviceKind != PointerDeviceKind.touch) {
+      return;
+    }
+    inputModel.tap(MouseButtons.left);
+    inputModel.tap(MouseButtons.left);
+  }
+
+  onLongPressDown(LongPressDownDetails d) {
+    lastDeviceKind = d.kind;
+    if (lastDeviceKind != PointerDeviceKind.touch) {
+      return;
+    }
+    if (handleTouch) {
+      ffi.cursorModel.move(d.localPosition.dx, d.localPosition.dy);
+      _cacheLongPressPosition = d.localPosition;
+    }
+  }
+
+  // for mobiles
+  onLongPress() {
+    if (lastDeviceKind != PointerDeviceKind.touch) {
+      return;
+    }
+    if (handleTouch) {
+      ffi.cursorModel
+          .move(_cacheLongPressPosition.dx, _cacheLongPressPosition.dy);
+    }
+    inputModel.tap(MouseButtons.right);
+  }
+
+  onDoubleFinerTap(TapDownDetails d) {
+    lastDeviceKind = d.kind;
+    if (lastDeviceKind != PointerDeviceKind.touch) {
+      return;
+    }
+    if (!handleTouch) {
+      inputModel.tap(MouseButtons.right);
+    }
+  }
+
+  onHoldDragStart(DragStartDetails d) {
+    lastDeviceKind = d.kind;
+    if (lastDeviceKind != PointerDeviceKind.touch) {
+      return;
+    }
+    if (!handleTouch) {
+      inputModel.sendMouse('down', MouseButtons.left);
+    }
+  }
+
+  onHoldDragUpdate(DragUpdateDetails d) {
+    if (lastDeviceKind != PointerDeviceKind.touch) {
+      return;
+    }
+    if (!handleTouch) {
+      ffi.cursorModel.updatePan(d.delta.dx, d.delta.dy, handleTouch);
+    }
+  }
+
+  onHoldDragEnd(DragEndDetails d) {
+    if (lastDeviceKind != PointerDeviceKind.touch) {
+      return;
+    }
+    if (!handleTouch) {
+      inputModel.sendMouse('up', MouseButtons.left);
+    }
+  }
+
+  onOneFingerPanStart(BuildContext context, DragStartDetails d) {
+    lastDeviceKind = d.kind ?? lastDeviceKind;
+    if (lastDeviceKind != PointerDeviceKind.touch) {
+      return;
+    }
+    if (handleTouch) {
+      ffi.cursorModel.move(d.localPosition.dx, d.localPosition.dy);
+      inputModel.sendMouse('down', MouseButtons.left);
+    } else {
+      final offset = ffi.cursorModel.offset;
+      final cursorX = offset.dx;
+      final cursorY = offset.dy;
+      final visible =
+          ffi.cursorModel.getVisibleRect().inflate(1); // extend edges
+      final size = MediaQueryData.fromView(View.of(context)).size;
+      if (!visible.contains(Offset(cursorX, cursorY))) {
+        ffi.cursorModel.move(size.width / 2, size.height / 2);
+      }
+    }
+  }
+
+  onOneFingerPanUpdate(DragUpdateDetails d) {
+    if (lastDeviceKind != PointerDeviceKind.touch) {
+      return;
+    }
+    ffi.cursorModel.updatePan(d.delta.dx, d.delta.dy, handleTouch);
+  }
+
+  onOneFingerPanEnd(DragEndDetails d) {
+    if (lastDeviceKind != PointerDeviceKind.touch) {
+      return;
+    }
+    if (handleTouch) {
+      inputModel.sendMouse('up', MouseButtons.left);
+    }
+  }
+
+  // scale + pan event
+  onTwoFingerScaleUpdate(ScaleUpdateDetails d) {
+    if (lastDeviceKind != PointerDeviceKind.touch) {
+      return;
+    }
+    if (isDesktop) {
+      // to-do
+    } else {
+      // mobile
+      // to-do: Is this correct?
+      ffi.canvasModel.updateScale(d.scale / _scale);
+      _scale = d.scale;
+      ffi.canvasModel.panX(d.focalPointDelta.dx);
+      ffi.canvasModel.panY(d.focalPointDelta.dy);
+    }
+  }
+
+  onTwoFingerScaleEnd(ScaleEndDetails d) {
+    if (lastDeviceKind != PointerDeviceKind.touch) {
+      return;
+    }
+    if (isDesktop) {
+      // to-do
+    } else {
+      // mobile
+      // to-do: Is this correct?
+      _scale = 1;
+      bind.sessionSetViewStyle(sessionId: sessionId, value: "");
+    }
+  }
+
+  get onHoldDragCancel => null;
+  get onThreeFingerVerticalDragUpdate => ffi.ffiModel.isPeerAndroid
+      ? null
+      : (d) {
+          _mouseScrollIntegral += d.delta.dy / 4;
+          if (_mouseScrollIntegral > 1) {
+            inputModel.scroll(1);
+            _mouseScrollIntegral = 0;
+          } else if (_mouseScrollIntegral < -1) {
+            inputModel.scroll(-1);
+            _mouseScrollIntegral = 0;
+          }
+        };
+
+  makeGestures(BuildContext context) {
+    return <Type, GestureRecognizerFactory>{
+      // Official
+      TapGestureRecognizer:
+          GestureRecognizerFactoryWithHandlers<TapGestureRecognizer>(
+              () => TapGestureRecognizer(), (instance) {
+        instance
+          ..onTapDown = onTapDown
+          ..onTapUp = onTapUp;
+      }),
+      DoubleTapGestureRecognizer:
+          GestureRecognizerFactoryWithHandlers<DoubleTapGestureRecognizer>(
+              () => DoubleTapGestureRecognizer(), (instance) {
+        instance
+          ..onDoubleTapDown = onDoubleTapDown
+          ..onDoubleTap = onDoubleTap;
+      }),
+      LongPressGestureRecognizer:
+          GestureRecognizerFactoryWithHandlers<LongPressGestureRecognizer>(
+              () => LongPressGestureRecognizer(), (instance) {
+        instance
+          ..onLongPressDown = onLongPressDown
+          ..onLongPress = onLongPress;
+      }),
+      // Customized
+      HoldTapMoveGestureRecognizer:
+          GestureRecognizerFactoryWithHandlers<HoldTapMoveGestureRecognizer>(
+              () => HoldTapMoveGestureRecognizer(),
+              (instance) => instance
+                ..onHoldDragStart = onHoldDragStart
+                ..onHoldDragUpdate = onHoldDragUpdate
+                ..onHoldDragCancel = onHoldDragCancel
+                ..onHoldDragEnd = onHoldDragEnd),
+      DoubleFinerTapGestureRecognizer:
+          GestureRecognizerFactoryWithHandlers<DoubleFinerTapGestureRecognizer>(
+              () => DoubleFinerTapGestureRecognizer(), (instance) {
+        instance.onDoubleFinerTap = onDoubleFinerTap;
+      }),
+      CustomTouchGestureRecognizer:
+          GestureRecognizerFactoryWithHandlers<CustomTouchGestureRecognizer>(
+              () => CustomTouchGestureRecognizer(), (instance) {
+        instance.onOneFingerPanStart =
+            (DragStartDetails d) => onOneFingerPanStart(context, d);
+        instance
+          ..onOneFingerPanUpdate = onOneFingerPanUpdate
+          ..onOneFingerPanEnd = onOneFingerPanEnd
+          ..onTwoFingerScaleUpdate = onTwoFingerScaleUpdate
+          ..onTwoFingerScaleEnd = onTwoFingerScaleEnd
+          ..onThreeFingerVerticalDragUpdate = onThreeFingerVerticalDragUpdate;
+      }),
+    };
+  }
+}
+
 class RawPointerMouseRegion extends StatelessWidget {
   final InputModel inputModel;
   final Widget child;
@@ -39,36 +329,39 @@ class RawPointerMouseRegion extends StatelessWidget {
   final PointerDownEventListener? onPointerDown;
   final PointerUpEventListener? onPointerUp;
 
-  RawPointerMouseRegion(
+  RawPointerMouseRegion({
-      {this.onEnter,
+    this.onEnter,
-      this.onExit,
+    this.onExit,
-      this.cursor,
+    this.cursor,
-      this.onPointerDown,
+    this.onPointerDown,
-      this.onPointerUp,
+    this.onPointerUp,
-      required this.inputModel,
+    required this.inputModel,
-      required this.child});
+    required this.child,
+  });
 
   @override
   Widget build(BuildContext context) {
     return Listener(
-        onPointerHover: inputModel.onPointHoverImage,
+      onPointerHover: inputModel.onPointHoverImage,
-        onPointerDown: (evt) {
+      onPointerDown: (evt) {
-          onPointerDown?.call(evt);
+        onPointerDown?.call(evt);
-          inputModel.onPointDownImage(evt);
+        inputModel.onPointDownImage(evt);
-        },
+      },
-        onPointerUp: (evt) {
+      onPointerUp: (evt) {
-          onPointerUp?.call(evt);
+        onPointerUp?.call(evt);
-          inputModel.onPointUpImage(evt);
+        inputModel.onPointUpImage(evt);
-        },
+      },
-        onPointerMove: inputModel.onPointMoveImage,
+      onPointerMove: inputModel.onPointMoveImage,
-        onPointerSignal: inputModel.onPointerSignalImage,
+      onPointerSignal: inputModel.onPointerSignalImage,
-        onPointerPanZoomStart: inputModel.onPointerPanZoomStart,
+      onPointerPanZoomStart: inputModel.onPointerPanZoomStart,
-        onPointerPanZoomUpdate: inputModel.onPointerPanZoomUpdate,
+      onPointerPanZoomUpdate: inputModel.onPointerPanZoomUpdate,
-        onPointerPanZoomEnd: inputModel.onPointerPanZoomEnd,
+      onPointerPanZoomEnd: inputModel.onPointerPanZoomEnd,
-        child: MouseRegion(
+      child: MouseRegion(
-            cursor: cursor ?? MouseCursor.defer,
+        cursor: cursor ?? MouseCursor.defer,
-            onEnter: onEnter,
+        onEnter: onEnter,
-            onExit: onExit,
+        onExit: onExit,
-            child: child));
+        child: child,
+      ),
+    );
   }
 }

flutter/lib/desktop/pages/remote_page.dart

@@ -337,6 +337,17 @@ class _RemotePageState extends State<RemotePage>
     }
   }
 
+  Widget _buildRawTouchAndPointerRegion(
+    Widget child,
+    PointerEnterEventListener? onEnter,
+    PointerExitEventListener? onExit,
+  ) {
+    return RawTouchGestureDetectorRegion(
+      child: _buildRawPointerMouseRegion(child, onEnter, onExit),
+      ffi: _ffi,
+    );
+  }
+
   Widget _buildRawPointerMouseRegion(
     Widget child,
     PointerEnterEventListener? onEnter,
@@ -384,7 +395,7 @@ class _RemotePageState extends State<RemotePage>
           textureId: _textureId,
           useTextureRender: useTextureRender,
           listenerBuilder: (child) =>
-              _buildRawPointerMouseRegion(child, enterView, leaveView),
+              _buildRawTouchAndPointerRegion(child, enterView, leaveView),
         );
       }))
     ];
@@ -401,7 +412,7 @@ class _RemotePageState extends State<RemotePage>
       Positioned(
         top: 10,
         right: 10,
-        child: _buildRawPointerMouseRegion(
+        child: _buildRawTouchAndPointerRegion(
             QualityMonitor(_ffi.qualityMonitorModel), null, null),
       ),
     );

flutter/lib/mobile/pages/remote_page.dart

@@ -1,5 +1,4 @@
 import 'dart:async';
-import 'dart:ui' as ui;
 
 import 'package:flutter/material.dart';
 import 'package:flutter/services.dart';
@@ -21,9 +20,8 @@ import '../../models/input_model.dart';
 import '../../models/model.dart';
 import '../../models/platform_model.dart';
 import '../../utils/image.dart';
-import '../widgets/gestures.dart';
 
-final initText = '\1' * 1024;
+final initText = '1' * 1024;
 
 class RemotePage extends StatefulWidget {
   RemotePage({Key? key, required this.id}) : super(key: key);
@@ -39,8 +37,6 @@ class _RemotePageState extends State<RemotePage> {
   bool _showBar = !isWebDesktop;
   bool _showGestureHelp = false;
   String _value = '';
-  double _scale = 1;
-  double _mouseScrollIntegral = 0; // mouse scroll speed controller
   Orientation? _currentOrientation;
 
   final keyboardVisibilityController = KeyboardVisibilityController();
@@ -267,11 +263,17 @@ class _RemotePageState extends State<RemotePage> {
                                 gFFI.canvasModel.updateViewStyle();
                               });
                             }
-                            return Obx(() => Container(
+                            return Obx(
+                              () => Container(
                                 color: MyTheme.canvasColor,
                                 child: inputModel.isPhysicalMouse.value
                                     ? getBodyForMobile()
-                                    : getBodyForMobileWithGesture()));
+                                    : RawTouchGestureDetectorRegion(
+                                        child: getBodyForMobile(),
+                                        ffi: gFFI,
+                                      ),
+                              ),
+                            );
                           })));
               })
             ],
@@ -377,120 +379,6 @@ class _RemotePageState extends State<RemotePage> {
     );
   }
 
-  /// touchMode only:
-  ///   LongPress -> right click
-  ///   OneFingerPan -> start/end -> left down start/end
-  ///   onDoubleTapDown -> move to
-  ///   onLongPressDown => move to
-  ///
-  /// mouseMode only:
-  ///   DoubleFiner -> right click
-  ///   HoldDrag -> left drag
-
-  Offset _cacheLongPressPosition = Offset(0, 0);
-  Widget getBodyForMobileWithGesture() {
-    final touchMode = gFFI.ffiModel.touchMode;
-    return getMixinGestureDetector(
-        child: getBodyForMobile(),
-        onTapUp: (d) {
-          if (touchMode) {
-            gFFI.cursorModel.move(d.localPosition.dx, d.localPosition.dy);
-            inputModel.tap(MouseButtons.left);
-          } else {
-            inputModel.tap(MouseButtons.left);
-          }
-        },
-        onDoubleTapDown: (d) {
-          if (touchMode) {
-            gFFI.cursorModel.move(d.localPosition.dx, d.localPosition.dy);
-          }
-        },
-        onDoubleTap: () {
-          inputModel.tap(MouseButtons.left);
-          inputModel.tap(MouseButtons.left);
-        },
-        onLongPressDown: (d) {
-          if (touchMode) {
-            gFFI.cursorModel.move(d.localPosition.dx, d.localPosition.dy);
-            _cacheLongPressPosition = d.localPosition;
-          }
-        },
-        onLongPress: () {
-          if (touchMode) {
-            gFFI.cursorModel
-                .move(_cacheLongPressPosition.dx, _cacheLongPressPosition.dy);
-          }
-          inputModel.tap(MouseButtons.right);
-        },
-        onDoubleFinerTap: (d) {
-          if (!touchMode) {
-            inputModel.tap(MouseButtons.right);
-          }
-        },
-        onHoldDragStart: (d) {
-          if (!touchMode) {
-            inputModel.sendMouse('down', MouseButtons.left);
-          }
-        },
-        onHoldDragUpdate: (d) {
-          if (!touchMode) {
-            gFFI.cursorModel.updatePan(d.delta.dx, d.delta.dy, touchMode);
-          }
-        },
-        onHoldDragEnd: (_) {
-          if (!touchMode) {
-            inputModel.sendMouse('up', MouseButtons.left);
-          }
-        },
-        onOneFingerPanStart: (d) {
-          if (touchMode) {
-            gFFI.cursorModel.move(d.localPosition.dx, d.localPosition.dy);
-            inputModel.sendMouse('down', MouseButtons.left);
-          } else {
-            final offset = gFFI.cursorModel.offset;
-            final cursorX = offset.dx;
-            final cursorY = offset.dy;
-            final visible =
-                gFFI.cursorModel.getVisibleRect().inflate(1); // extend edges
-            final size = MediaQueryData.fromWindow(ui.window).size;
-            if (!visible.contains(Offset(cursorX, cursorY))) {
-              gFFI.cursorModel.move(size.width / 2, size.height / 2);
-            }
-          }
-        },
-        onOneFingerPanUpdate: (d) {
-          gFFI.cursorModel.updatePan(d.delta.dx, d.delta.dy, touchMode);
-        },
-        onOneFingerPanEnd: (d) {
-          if (touchMode) {
-            inputModel.sendMouse('up', MouseButtons.left);
-          }
-        },
-        // scale + pan event
-        onTwoFingerScaleUpdate: (d) {
-          gFFI.canvasModel.updateScale(d.scale / _scale);
-          _scale = d.scale;
-          gFFI.canvasModel.panX(d.focalPointDelta.dx);
-          gFFI.canvasModel.panY(d.focalPointDelta.dy);
-        },
-        onTwoFingerScaleEnd: (d) {
-          _scale = 1;
-          bind.sessionSetViewStyle(sessionId: sessionId, value: "");
-        },
-        onThreeFingerVerticalDragUpdate: gFFI.ffiModel.isPeerAndroid
-            ? null
-            : (d) {
-                _mouseScrollIntegral += d.delta.dy / 4;
-                if (_mouseScrollIntegral > 1) {
-                  inputModel.scroll(1);
-                  _mouseScrollIntegral = 0;
-                } else if (_mouseScrollIntegral < -1) {
-                  inputModel.scroll(-1);
-                  _mouseScrollIntegral = 0;
-                }
-              });
-  }
-
   Widget getBodyForMobile() {
     final keyboardIsVisible = keyboardVisibilityController.isVisible;
     return Container(

flutter/lib/models/input_model.dart

@@ -268,6 +268,14 @@ class InputModel {
     sendMouse('up', button);
   }
 
+  void tapDown(MouseButtons button) {
+    sendMouse('down', button);
+  }
+
+    void tapUp(MouseButtons button) {
+    sendMouse('up', button);
+  }
+
   /// Send scroll event with scroll distance [y].
   void scroll(int y) {
     bind.sessionSendMouse(
@@ -429,7 +437,7 @@ class InputModel {
   }
 
   void onPointDownImage(PointerDownEvent e) {
-    debugPrint("onPointDownImage");
+    debugPrint("onPointDownImage ${e.kind}");
     _stopFling = true;
     if (e.kind != ui.PointerDeviceKind.mouse) {
       if (isPhysicalMouse.value) {
@@ -469,6 +477,7 @@ class InputModel {
       } else if (dy < 0) {
         dy = 1;
       }
+      debugPrint('REMOVE ME ================== onPointerSignalImage');
       bind.sessionSendMouse(
           sessionId: sessionId,
           msg: '{"type": "wheel", "x": "$dx", "y": "$dy"}');

flutter/lib/models/model.dart

@@ -1265,7 +1265,6 @@ class CursorModel with ChangeNotifier {
   }
 
   updatePan(double dx, double dy, bool touchMode) {
-    if (parent.target?.imageModel.image == null) return;
     if (touchMode) {
       final scale = parent.target?.canvasModel.scale ?? 1.0;
       _x += dx / scale;
@@ -1274,6 +1273,7 @@ class CursorModel with ChangeNotifier {
       notifyListeners();
       return;
     }
+    if (parent.target?.imageModel.image == null) return;
     final scale = parent.target?.canvasModel.scale ?? 1.0;
     dx /= scale;
     dy /= scale;