From a548e9c94dba8747e19a10d3987d7ebac96e0f49 Mon Sep 17 00:00:00 2001 From: fufesou <13586388+fufesou@users.noreply.github.com> Date: Sat, 15 Feb 2025 18:33:26 +0800 Subject: [PATCH] fix: android, controlled side, gesture (#10792) Signed-off-by: fufesou --- .../com/carriez/flutter_hbb/InputService.kt | 81 ++++++++++++------- .../com/carriez/flutter_hbb/MainService.kt | 4 +- flutter/lib/common/widgets/remote_input.dart | 38 +++++++-- 3 files changed, 85 insertions(+), 38 deletions(-) diff --git a/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt b/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt index f53f95d67..b510be35b 100644 --- a/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt +++ b/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt @@ -34,9 +34,9 @@ import hbb.MessageOuterClass.KeyEvent import hbb.MessageOuterClass.KeyboardMode import hbb.KeyEventConverter -const val LIFT_DOWN = 9 -const val LIFT_MOVE = 8 -const val LIFT_UP = 10 +const val LEFT_DOWN = 9 +const val LEFT_MOVE = 8 +const val LEFT_UP = 10 const val RIGHT_UP = 18 const val WHEEL_BUTTON_DOWN = 33 const val WHEEL_BUTTON_UP = 34 @@ -65,6 +65,7 @@ class InputService : AccessibilityService() { private val logTag = "input service" private var leftIsDown = false private var touchPath = Path() + private var stroke: GestureDescription.StrokeDescription? = null private var lastTouchGestureStartTime = 0L private var mouseX = 0 private var mouseY = 0 @@ -77,6 +78,9 @@ class InputService : AccessibilityService() { private var fakeEditTextForTextStateCalculation: EditText? = null + private var lastX = 0 + private var lastY = 0 + private val volumeController: VolumeController by lazy { VolumeController(applicationContext.getSystemService(AUDIO_SERVICE) as AudioManager) } @RequiresApi(Build.VERSION_CODES.N) @@ -84,7 +88,7 @@ class InputService : AccessibilityService() { val x = max(0, _x) val y = max(0, _y) - if (mask == 0 || mask == LIFT_MOVE) { + if (mask == 0 || mask == LEFT_MOVE) { val oldX = mouseX val oldY = mouseY mouseX = x * SCREEN_INFO.scale @@ -99,14 +103,13 @@ class InputService : AccessibilityService() { } // left button down ,was up - if (mask == LIFT_DOWN) { + if (mask == LEFT_DOWN) { isWaitingLongPress = true timer.schedule(object : TimerTask() { override fun run() { if (isWaitingLongPress) { isWaitingLongPress = false - leftIsDown = false - endGesture(mouseX, mouseY) + continueGesture(mouseX, mouseY) } } }, LONG_TAP_DELAY * 4) @@ -122,7 +125,7 @@ class InputService : AccessibilityService() { } // left up ,was down - if (mask == LIFT_UP) { + if (mask == LEFT_UP) { if (leftIsDown) { leftIsDown = false isWaitingLongPress = false @@ -242,35 +245,57 @@ class InputService : AccessibilityService() { } private fun startGesture(x: Int, y: Int) { - touchPath = Path() + touchPath.reset() touchPath.moveTo(x.toFloat(), y.toFloat()) lastTouchGestureStartTime = System.currentTimeMillis() + lastX = x + lastY = y } - private fun continueGesture(x: Int, y: Int) { + @RequiresApi(Build.VERSION_CODES.N) + private fun doDispatchGesture(x: Int, y: Int, willContinue: Boolean) { touchPath.lineTo(x.toFloat(), y.toFloat()) + var duration = System.currentTimeMillis() - lastTouchGestureStartTime + if (duration <= 0) { + duration = 1 + } + try { + if (stroke == null) { + stroke = GestureDescription.StrokeDescription( + touchPath, + 0, + duration, + willContinue + ) + } else { + stroke = stroke?.continueStroke(touchPath, 0, duration, willContinue) + } + stroke?.let { + val builder = GestureDescription.Builder() + builder.addStroke(it) + Log.d(logTag, "end gesture x:$x y:$y time:$duration") + dispatchGesture(builder.build(), null, null) + } + } catch (e: Exception) { + Log.e(logTag, "doDispatchGesture, willContinue:$willContinue, error:$e") + } + } + + @RequiresApi(Build.VERSION_CODES.N) + private fun continueGesture(x: Int, y: Int) { + doDispatchGesture(x, y, true) + touchPath.reset() + touchPath.moveTo(x.toFloat(), y.toFloat()) + lastTouchGestureStartTime = System.currentTimeMillis() + lastX = x + lastY = y } @RequiresApi(Build.VERSION_CODES.N) private fun endGesture(x: Int, y: Int) { - try { - touchPath.lineTo(x.toFloat(), y.toFloat()) - var duration = System.currentTimeMillis() - lastTouchGestureStartTime - if (duration <= 0) { - duration = 1 - } - val stroke = GestureDescription.StrokeDescription( - touchPath, - 0, - duration - ) - val builder = GestureDescription.Builder() - builder.addStroke(stroke) - Log.d(logTag, "end gesture x:$x y:$y time:$duration") - dispatchGesture(builder.build(), null, null) - } catch (e: Exception) { - Log.e(logTag, "endGesture error:$e") - } + doDispatchGesture(x, y, false) + touchPath.reset() + stroke = null } @RequiresApi(Build.VERSION_CODES.N) diff --git a/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/MainService.kt b/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/MainService.kt index e9ec0975d..c475dd3b0 100644 --- a/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/MainService.kt +++ b/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/MainService.kt @@ -65,8 +65,8 @@ class MainService : Service() { @Keep @RequiresApi(Build.VERSION_CODES.N) fun rustPointerInput(kind: Int, mask: Int, x: Int, y: Int) { - // turn on screen with LIFT_DOWN when screen off - if (!powerManager.isInteractive && (kind == 0 || mask == LIFT_DOWN)) { + // turn on screen with LEFT_DOWN when screen off + if (!powerManager.isInteractive && (kind == 0 || mask == LEFT_DOWN)) { if (wakeLock.isHeld) { Log.d(logTag, "Turn on Screen, WakeLock release") wakeLock.release() diff --git a/flutter/lib/common/widgets/remote_input.dart b/flutter/lib/common/widgets/remote_input.dart index 6eb9b0594..6667bdf80 100644 --- a/flutter/lib/common/widgets/remote_input.dart +++ b/flutter/lib/common/widgets/remote_input.dart @@ -187,6 +187,11 @@ class _RawTouchGestureDetectorRegionState return; } _cacheLongPressPositionTs = DateTime.now().millisecondsSinceEpoch; + if (ffiModel.isPeerMobile) { + await ffi.cursorModel + .move(_cacheLongPressPosition.dx, _cacheLongPressPosition.dy); + await inputModel.tapDown(MouseButtons.left); + } } } @@ -204,15 +209,31 @@ class _RawTouchGestureDetectorRegionState if (lastDeviceKind != PointerDeviceKind.touch) { return; } + if (!ffi.ffiModel.isPeerMobile) { + if (handleTouch) { + final isMoved = await ffi.cursorModel + .move(_cacheLongPressPosition.dx, _cacheLongPressPosition.dy); + if (!isMoved) { + return; + } + } + await inputModel.tap(MouseButtons.right); + } else { + // It's better to send a message to tell the controlled device that the long press event is triggered. + // We're now using a `TimerTask` in `InputService.kt` to decide whether to trigger the long press event. + // It's not accurate and it's better to use the same detection logic in the controlling side. + } + } + + onLongPressMoveUpdate(LongPressMoveUpdateDetails d) async { + if (!ffiModel.isPeerMobile || lastDeviceKind != PointerDeviceKind.touch) { + return; + } if (handleTouch) { - final isMoved = await ffi.cursorModel - .move(_cacheLongPressPosition.dx, _cacheLongPressPosition.dy); - if (!isMoved) { + if (!ffi.cursorModel.isInRemoteRect(d.localPosition)) { return; } - } - if (!ffi.ffiModel.isPeerMobile) { - await inputModel.tap(MouseButtons.right); + await ffi.cursorModel.move(d.localPosition.dx, d.localPosition.dy); } } @@ -340,7 +361,7 @@ class _RawTouchGestureDetectorRegionState ffi.cursorModel.clearRemoteWindowCoords(); } if (handleTouch) { - await inputModel.sendMouse('up', MouseButtons.left); + await inputModel.sendMouse('up', MouseButtons.left); } } @@ -432,7 +453,8 @@ class _RawTouchGestureDetectorRegionState instance ..onLongPressDown = onLongPressDown ..onLongPressUp = onLongPressUp - ..onLongPress = onLongPress; + ..onLongPress = onLongPress + ..onLongPressMoveUpdate = onLongPressMoveUpdate; }), // Customized HoldTapMoveGestureRecognizer: