From c01c8d0afc094bd1e2164c19eb0966aedea9de78 Mon Sep 17 00:00:00 2001 From: dignow Date: Mon, 17 Jul 2023 20:07:55 +0800 Subject: [PATCH] touch screen input Signed-off-by: dignow --- .../{mobile => common}/widgets/gestures.dart | 0 flutter/lib/common/widgets/remote_input.dart | 349 ++++++++++++++++-- flutter/lib/desktop/pages/remote_page.dart | 15 +- flutter/lib/mobile/pages/remote_page.dart | 130 +------ flutter/lib/models/input_model.dart | 11 +- flutter/lib/models/model.dart | 2 +- 6 files changed, 354 insertions(+), 153 deletions(-) rename flutter/lib/{mobile => common}/widgets/gestures.dart (100%) diff --git a/flutter/lib/mobile/widgets/gestures.dart b/flutter/lib/common/widgets/gestures.dart similarity index 100% rename from flutter/lib/mobile/widgets/gestures.dart rename to flutter/lib/common/widgets/gestures.dart diff --git a/flutter/lib/common/widgets/remote_input.dart b/flutter/lib/common/widgets/remote_input.dart index dd39cbdfd..5fc37e939 100644 --- a/flutter/lib/common/widgets/remote_input.dart +++ b/flutter/lib/common/widgets/remote_input.dart @@ -1,7 +1,13 @@ import 'package:flutter/material.dart'; import 'package:flutter/services.dart'; +import 'package:flutter/gestures.dart'; -import '../../models/input_model.dart'; +import 'package:flutter_hbb/models/platform_model.dart'; +import 'package:flutter_hbb/common.dart'; +import 'package:flutter_hbb/models/model.dart'; +import 'package:flutter_hbb/models/input_model.dart'; + +import './gestures.dart'; class RawKeyFocusScope extends StatelessWidget { final FocusNode? focusNode; @@ -30,6 +36,290 @@ class RawKeyFocusScope extends StatelessWidget { } } +class RawTouchGestureDetectorRegion extends StatefulWidget { + final Widget child; + final FFI ffi; + + late final InputModel inputModel = ffi.inputModel; + late final FfiModel ffiModel = ffi.ffiModel; + + RawTouchGestureDetectorRegion({ + required this.child, + required this.ffi, + }); + + @override + State createState() => + _RawTouchGestureDetectorRegionState(); +} + + /// touchMode only: + /// LongPress -> right click + /// OneFingerPan -> start/end -> left down start/end + /// onDoubleTapDown -> move to + /// onLongPressDown => move to + /// + /// mouseMode only: + /// DoubleFiner -> right click + /// HoldDrag -> left drag +class _RawTouchGestureDetectorRegionState + extends State { + Offset _cacheLongPressPosition = Offset(0, 0); + double _mouseScrollIntegral = 0; // mouse scroll speed controller + double _scale = 1; + + PointerDeviceKind? lastDeviceKind; + + FFI get ffi => widget.ffi; + FfiModel get ffiModel => widget.ffiModel; + InputModel get inputModel => widget.inputModel; + bool get handleTouch => isDesktop || ffiModel.touchMode; + SessionID get sessionId => ffi.sessionId; + + @override + Widget build(BuildContext context) { + return RawGestureDetector( + child: widget.child, + gestures: makeGestures(context), + ); + } + + onTapDown(TapDownDetails d) { + lastDeviceKind = d.kind; + if (lastDeviceKind != PointerDeviceKind.touch) { + return; + } + if (handleTouch) { + ffi.cursorModel.move(d.localPosition.dx, d.localPosition.dy); + } + inputModel.tapDown(MouseButtons.left); + } + + onTapUp(TapUpDetails d) { + if (lastDeviceKind != PointerDeviceKind.touch) { + return; + } + if (handleTouch) { + ffi.cursorModel.move(d.localPosition.dx, d.localPosition.dy); + } + inputModel.tapUp(MouseButtons.left); + } + + onDoubleTapDown(TapDownDetails d) { + lastDeviceKind = d.kind; + if (lastDeviceKind != PointerDeviceKind.touch) { + return; + } + if (handleTouch) { + ffi.cursorModel.move(d.localPosition.dx, d.localPosition.dy); + } + } + + onDoubleTap() { + if (lastDeviceKind != PointerDeviceKind.touch) { + return; + } + inputModel.tap(MouseButtons.left); + inputModel.tap(MouseButtons.left); + } + + onLongPressDown(LongPressDownDetails d) { + lastDeviceKind = d.kind; + if (lastDeviceKind != PointerDeviceKind.touch) { + return; + } + if (handleTouch) { + ffi.cursorModel.move(d.localPosition.dx, d.localPosition.dy); + _cacheLongPressPosition = d.localPosition; + } + } + + // for mobiles + onLongPress() { + if (lastDeviceKind != PointerDeviceKind.touch) { + return; + } + if (handleTouch) { + ffi.cursorModel + .move(_cacheLongPressPosition.dx, _cacheLongPressPosition.dy); + } + inputModel.tap(MouseButtons.right); + } + + onDoubleFinerTap(TapDownDetails d) { + lastDeviceKind = d.kind; + if (lastDeviceKind != PointerDeviceKind.touch) { + return; + } + if (!handleTouch) { + inputModel.tap(MouseButtons.right); + } + } + + onHoldDragStart(DragStartDetails d) { + lastDeviceKind = d.kind; + if (lastDeviceKind != PointerDeviceKind.touch) { + return; + } + if (!handleTouch) { + inputModel.sendMouse('down', MouseButtons.left); + } + } + + onHoldDragUpdate(DragUpdateDetails d) { + if (lastDeviceKind != PointerDeviceKind.touch) { + return; + } + if (!handleTouch) { + ffi.cursorModel.updatePan(d.delta.dx, d.delta.dy, handleTouch); + } + } + + onHoldDragEnd(DragEndDetails d) { + if (lastDeviceKind != PointerDeviceKind.touch) { + return; + } + if (!handleTouch) { + inputModel.sendMouse('up', MouseButtons.left); + } + } + + onOneFingerPanStart(BuildContext context, DragStartDetails d) { + lastDeviceKind = d.kind ?? lastDeviceKind; + if (lastDeviceKind != PointerDeviceKind.touch) { + return; + } + if (handleTouch) { + ffi.cursorModel.move(d.localPosition.dx, d.localPosition.dy); + inputModel.sendMouse('down', MouseButtons.left); + } else { + final offset = ffi.cursorModel.offset; + final cursorX = offset.dx; + final cursorY = offset.dy; + final visible = + ffi.cursorModel.getVisibleRect().inflate(1); // extend edges + final size = MediaQueryData.fromView(View.of(context)).size; + if (!visible.contains(Offset(cursorX, cursorY))) { + ffi.cursorModel.move(size.width / 2, size.height / 2); + } + } + } + + onOneFingerPanUpdate(DragUpdateDetails d) { + if (lastDeviceKind != PointerDeviceKind.touch) { + return; + } + ffi.cursorModel.updatePan(d.delta.dx, d.delta.dy, handleTouch); + } + + onOneFingerPanEnd(DragEndDetails d) { + if (lastDeviceKind != PointerDeviceKind.touch) { + return; + } + if (handleTouch) { + inputModel.sendMouse('up', MouseButtons.left); + } + } + + // scale + pan event + onTwoFingerScaleUpdate(ScaleUpdateDetails d) { + if (lastDeviceKind != PointerDeviceKind.touch) { + return; + } + if (isDesktop) { + // to-do + } else { + // mobile + // to-do: Is this correct? + ffi.canvasModel.updateScale(d.scale / _scale); + _scale = d.scale; + ffi.canvasModel.panX(d.focalPointDelta.dx); + ffi.canvasModel.panY(d.focalPointDelta.dy); + } + } + + onTwoFingerScaleEnd(ScaleEndDetails d) { + if (lastDeviceKind != PointerDeviceKind.touch) { + return; + } + if (isDesktop) { + // to-do + } else { + // mobile + // to-do: Is this correct? + _scale = 1; + bind.sessionSetViewStyle(sessionId: sessionId, value: ""); + } + } + + get onHoldDragCancel => null; + get onThreeFingerVerticalDragUpdate => ffi.ffiModel.isPeerAndroid + ? null + : (d) { + _mouseScrollIntegral += d.delta.dy / 4; + if (_mouseScrollIntegral > 1) { + inputModel.scroll(1); + _mouseScrollIntegral = 0; + } else if (_mouseScrollIntegral < -1) { + inputModel.scroll(-1); + _mouseScrollIntegral = 0; + } + }; + + makeGestures(BuildContext context) { + return { + // Official + TapGestureRecognizer: + GestureRecognizerFactoryWithHandlers( + () => TapGestureRecognizer(), (instance) { + instance + ..onTapDown = onTapDown + ..onTapUp = onTapUp; + }), + DoubleTapGestureRecognizer: + GestureRecognizerFactoryWithHandlers( + () => DoubleTapGestureRecognizer(), (instance) { + instance + ..onDoubleTapDown = onDoubleTapDown + ..onDoubleTap = onDoubleTap; + }), + LongPressGestureRecognizer: + GestureRecognizerFactoryWithHandlers( + () => LongPressGestureRecognizer(), (instance) { + instance + ..onLongPressDown = onLongPressDown + ..onLongPress = onLongPress; + }), + // Customized + HoldTapMoveGestureRecognizer: + GestureRecognizerFactoryWithHandlers( + () => HoldTapMoveGestureRecognizer(), + (instance) => instance + ..onHoldDragStart = onHoldDragStart + ..onHoldDragUpdate = onHoldDragUpdate + ..onHoldDragCancel = onHoldDragCancel + ..onHoldDragEnd = onHoldDragEnd), + DoubleFinerTapGestureRecognizer: + GestureRecognizerFactoryWithHandlers( + () => DoubleFinerTapGestureRecognizer(), (instance) { + instance.onDoubleFinerTap = onDoubleFinerTap; + }), + CustomTouchGestureRecognizer: + GestureRecognizerFactoryWithHandlers( + () => CustomTouchGestureRecognizer(), (instance) { + instance.onOneFingerPanStart = + (DragStartDetails d) => onOneFingerPanStart(context, d); + instance + ..onOneFingerPanUpdate = onOneFingerPanUpdate + ..onOneFingerPanEnd = onOneFingerPanEnd + ..onTwoFingerScaleUpdate = onTwoFingerScaleUpdate + ..onTwoFingerScaleEnd = onTwoFingerScaleEnd + ..onThreeFingerVerticalDragUpdate = onThreeFingerVerticalDragUpdate; + }), + }; + } +} + class RawPointerMouseRegion extends StatelessWidget { final InputModel inputModel; final Widget child; @@ -39,36 +329,39 @@ class RawPointerMouseRegion extends StatelessWidget { final PointerDownEventListener? onPointerDown; final PointerUpEventListener? onPointerUp; - RawPointerMouseRegion( - {this.onEnter, - this.onExit, - this.cursor, - this.onPointerDown, - this.onPointerUp, - required this.inputModel, - required this.child}); + RawPointerMouseRegion({ + this.onEnter, + this.onExit, + this.cursor, + this.onPointerDown, + this.onPointerUp, + required this.inputModel, + required this.child, + }); @override Widget build(BuildContext context) { return Listener( - onPointerHover: inputModel.onPointHoverImage, - onPointerDown: (evt) { - onPointerDown?.call(evt); - inputModel.onPointDownImage(evt); - }, - onPointerUp: (evt) { - onPointerUp?.call(evt); - inputModel.onPointUpImage(evt); - }, - onPointerMove: inputModel.onPointMoveImage, - onPointerSignal: inputModel.onPointerSignalImage, - onPointerPanZoomStart: inputModel.onPointerPanZoomStart, - onPointerPanZoomUpdate: inputModel.onPointerPanZoomUpdate, - onPointerPanZoomEnd: inputModel.onPointerPanZoomEnd, - child: MouseRegion( - cursor: cursor ?? MouseCursor.defer, - onEnter: onEnter, - onExit: onExit, - child: child)); + onPointerHover: inputModel.onPointHoverImage, + onPointerDown: (evt) { + onPointerDown?.call(evt); + inputModel.onPointDownImage(evt); + }, + onPointerUp: (evt) { + onPointerUp?.call(evt); + inputModel.onPointUpImage(evt); + }, + onPointerMove: inputModel.onPointMoveImage, + onPointerSignal: inputModel.onPointerSignalImage, + onPointerPanZoomStart: inputModel.onPointerPanZoomStart, + onPointerPanZoomUpdate: inputModel.onPointerPanZoomUpdate, + onPointerPanZoomEnd: inputModel.onPointerPanZoomEnd, + child: MouseRegion( + cursor: cursor ?? MouseCursor.defer, + onEnter: onEnter, + onExit: onExit, + child: child, + ), + ); } } diff --git a/flutter/lib/desktop/pages/remote_page.dart b/flutter/lib/desktop/pages/remote_page.dart index 849971a41..35705a283 100644 --- a/flutter/lib/desktop/pages/remote_page.dart +++ b/flutter/lib/desktop/pages/remote_page.dart @@ -337,6 +337,17 @@ class _RemotePageState extends State } } + Widget _buildRawTouchAndPointerRegion( + Widget child, + PointerEnterEventListener? onEnter, + PointerExitEventListener? onExit, + ) { + return RawTouchGestureDetectorRegion( + child: _buildRawPointerMouseRegion(child, onEnter, onExit), + ffi: _ffi, + ); + } + Widget _buildRawPointerMouseRegion( Widget child, PointerEnterEventListener? onEnter, @@ -384,7 +395,7 @@ class _RemotePageState extends State textureId: _textureId, useTextureRender: useTextureRender, listenerBuilder: (child) => - _buildRawPointerMouseRegion(child, enterView, leaveView), + _buildRawTouchAndPointerRegion(child, enterView, leaveView), ); })) ]; @@ -401,7 +412,7 @@ class _RemotePageState extends State Positioned( top: 10, right: 10, - child: _buildRawPointerMouseRegion( + child: _buildRawTouchAndPointerRegion( QualityMonitor(_ffi.qualityMonitorModel), null, null), ), ); diff --git a/flutter/lib/mobile/pages/remote_page.dart b/flutter/lib/mobile/pages/remote_page.dart index c6fc1baa1..84426a307 100644 --- a/flutter/lib/mobile/pages/remote_page.dart +++ b/flutter/lib/mobile/pages/remote_page.dart @@ -1,5 +1,4 @@ import 'dart:async'; -import 'dart:ui' as ui; import 'package:flutter/material.dart'; import 'package:flutter/services.dart'; @@ -21,9 +20,8 @@ import '../../models/input_model.dart'; import '../../models/model.dart'; import '../../models/platform_model.dart'; import '../../utils/image.dart'; -import '../widgets/gestures.dart'; -final initText = '\1' * 1024; +final initText = '1' * 1024; class RemotePage extends StatefulWidget { RemotePage({Key? key, required this.id}) : super(key: key); @@ -39,8 +37,6 @@ class _RemotePageState extends State { bool _showBar = !isWebDesktop; bool _showGestureHelp = false; String _value = ''; - double _scale = 1; - double _mouseScrollIntegral = 0; // mouse scroll speed controller Orientation? _currentOrientation; final keyboardVisibilityController = KeyboardVisibilityController(); @@ -267,11 +263,17 @@ class _RemotePageState extends State { gFFI.canvasModel.updateViewStyle(); }); } - return Obx(() => Container( + return Obx( + () => Container( color: MyTheme.canvasColor, child: inputModel.isPhysicalMouse.value ? getBodyForMobile() - : getBodyForMobileWithGesture())); + : RawTouchGestureDetectorRegion( + child: getBodyForMobile(), + ffi: gFFI, + ), + ), + ); }))); }) ], @@ -377,120 +379,6 @@ class _RemotePageState extends State { ); } - /// touchMode only: - /// LongPress -> right click - /// OneFingerPan -> start/end -> left down start/end - /// onDoubleTapDown -> move to - /// onLongPressDown => move to - /// - /// mouseMode only: - /// DoubleFiner -> right click - /// HoldDrag -> left drag - - Offset _cacheLongPressPosition = Offset(0, 0); - Widget getBodyForMobileWithGesture() { - final touchMode = gFFI.ffiModel.touchMode; - return getMixinGestureDetector( - child: getBodyForMobile(), - onTapUp: (d) { - if (touchMode) { - gFFI.cursorModel.move(d.localPosition.dx, d.localPosition.dy); - inputModel.tap(MouseButtons.left); - } else { - inputModel.tap(MouseButtons.left); - } - }, - onDoubleTapDown: (d) { - if (touchMode) { - gFFI.cursorModel.move(d.localPosition.dx, d.localPosition.dy); - } - }, - onDoubleTap: () { - inputModel.tap(MouseButtons.left); - inputModel.tap(MouseButtons.left); - }, - onLongPressDown: (d) { - if (touchMode) { - gFFI.cursorModel.move(d.localPosition.dx, d.localPosition.dy); - _cacheLongPressPosition = d.localPosition; - } - }, - onLongPress: () { - if (touchMode) { - gFFI.cursorModel - .move(_cacheLongPressPosition.dx, _cacheLongPressPosition.dy); - } - inputModel.tap(MouseButtons.right); - }, - onDoubleFinerTap: (d) { - if (!touchMode) { - inputModel.tap(MouseButtons.right); - } - }, - onHoldDragStart: (d) { - if (!touchMode) { - inputModel.sendMouse('down', MouseButtons.left); - } - }, - onHoldDragUpdate: (d) { - if (!touchMode) { - gFFI.cursorModel.updatePan(d.delta.dx, d.delta.dy, touchMode); - } - }, - onHoldDragEnd: (_) { - if (!touchMode) { - inputModel.sendMouse('up', MouseButtons.left); - } - }, - onOneFingerPanStart: (d) { - if (touchMode) { - gFFI.cursorModel.move(d.localPosition.dx, d.localPosition.dy); - inputModel.sendMouse('down', MouseButtons.left); - } else { - final offset = gFFI.cursorModel.offset; - final cursorX = offset.dx; - final cursorY = offset.dy; - final visible = - gFFI.cursorModel.getVisibleRect().inflate(1); // extend edges - final size = MediaQueryData.fromWindow(ui.window).size; - if (!visible.contains(Offset(cursorX, cursorY))) { - gFFI.cursorModel.move(size.width / 2, size.height / 2); - } - } - }, - onOneFingerPanUpdate: (d) { - gFFI.cursorModel.updatePan(d.delta.dx, d.delta.dy, touchMode); - }, - onOneFingerPanEnd: (d) { - if (touchMode) { - inputModel.sendMouse('up', MouseButtons.left); - } - }, - // scale + pan event - onTwoFingerScaleUpdate: (d) { - gFFI.canvasModel.updateScale(d.scale / _scale); - _scale = d.scale; - gFFI.canvasModel.panX(d.focalPointDelta.dx); - gFFI.canvasModel.panY(d.focalPointDelta.dy); - }, - onTwoFingerScaleEnd: (d) { - _scale = 1; - bind.sessionSetViewStyle(sessionId: sessionId, value: ""); - }, - onThreeFingerVerticalDragUpdate: gFFI.ffiModel.isPeerAndroid - ? null - : (d) { - _mouseScrollIntegral += d.delta.dy / 4; - if (_mouseScrollIntegral > 1) { - inputModel.scroll(1); - _mouseScrollIntegral = 0; - } else if (_mouseScrollIntegral < -1) { - inputModel.scroll(-1); - _mouseScrollIntegral = 0; - } - }); - } - Widget getBodyForMobile() { final keyboardIsVisible = keyboardVisibilityController.isVisible; return Container( diff --git a/flutter/lib/models/input_model.dart b/flutter/lib/models/input_model.dart index e7ce5c585..0ea7c6238 100644 --- a/flutter/lib/models/input_model.dart +++ b/flutter/lib/models/input_model.dart @@ -268,6 +268,14 @@ class InputModel { sendMouse('up', button); } + void tapDown(MouseButtons button) { + sendMouse('down', button); + } + + void tapUp(MouseButtons button) { + sendMouse('up', button); + } + /// Send scroll event with scroll distance [y]. void scroll(int y) { bind.sessionSendMouse( @@ -429,7 +437,7 @@ class InputModel { } void onPointDownImage(PointerDownEvent e) { - debugPrint("onPointDownImage"); + debugPrint("onPointDownImage ${e.kind}"); _stopFling = true; if (e.kind != ui.PointerDeviceKind.mouse) { if (isPhysicalMouse.value) { @@ -469,6 +477,7 @@ class InputModel { } else if (dy < 0) { dy = 1; } + debugPrint('REMOVE ME ================== onPointerSignalImage'); bind.sessionSendMouse( sessionId: sessionId, msg: '{"type": "wheel", "x": "$dx", "y": "$dy"}'); diff --git a/flutter/lib/models/model.dart b/flutter/lib/models/model.dart index 4eeb8e84f..54965680a 100644 --- a/flutter/lib/models/model.dart +++ b/flutter/lib/models/model.dart @@ -1265,7 +1265,6 @@ class CursorModel with ChangeNotifier { } updatePan(double dx, double dy, bool touchMode) { - if (parent.target?.imageModel.image == null) return; if (touchMode) { final scale = parent.target?.canvasModel.scale ?? 1.0; _x += dx / scale; @@ -1274,6 +1273,7 @@ class CursorModel with ChangeNotifier { notifyListeners(); return; } + if (parent.target?.imageModel.image == null) return; final scale = parent.target?.canvasModel.scale ?? 1.0; dx /= scale; dy /= scale;