feat: legacy mode android keyboard support

2025-12-12 11:06:57 +00:00 · 2023-10-19 00:16:22 +08:00
parent bbc241748b
commit 22165ec1a5
8 changed files with 261 additions and 28 deletions
--- a/flutter/android/app/build.gradle
+++ b/flutter/android/app/build.gradle
@@ -1,3 +1,8 @@
+import com.google.protobuf.gradle.*
+plugins {
+    id "com.google.protobuf" version "0.9.4"
+}
+
 def keystoreProperties = new Properties()
 def keystorePropertiesFile = rootProject.file('key.properties')
 if (keystorePropertiesFile.exists()) {
@@ -31,10 +36,33 @@ apply plugin: 'com.android.application'
 apply plugin: 'kotlin-android'
 apply from: "$flutterRoot/packages/flutter_tools/gradle/flutter.gradle"

+dependencies {
+    implementation 'com.google.protobuf:protobuf-javalite:3.20.1'
+}
+
+protobuf {
+    protoc {
+        artifact = 'com.google.protobuf:protoc:3.20.1'
+    }
+
+    generateProtoTasks {
+        all().configureEach { task ->
+            task.builtins {
+                java {
+                    option "lite"
+                }
+            }
+        }
+    }
+}
+
 android {
    compileSdkVersion 33
    sourceSets {
        main.java.srcDirs += 'src/main/kotlin'
+
+        main.proto.srcDirs += '../../../libs/hbb_common/protos'
+        main.proto.includes += "message.proto"
    }

    compileOptions {
--- a/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt
+++ b/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt
@@ -11,13 +11,18 @@ import android.accessibilityservice.GestureDescription
 import android.graphics.Path
 import android.os.Build
 import android.os.Bundle
+import android.os.Handler
+import android.os.Looper
 import android.util.Log
+import android.widget.EditText
 import android.view.accessibility.AccessibilityEvent
 import android.view.accessibility.AccessibilityNodeInfo
 import androidx.annotation.RequiresApi
 import java.util.*
 import kotlin.math.abs
 import kotlin.math.max
+import hbb.MessageOuterClass.KeyEvent;
+import hbb.KeyEventConverter;

 const val LIFT_DOWN = 9
 const val LIFT_MOVE = 8
@@ -60,6 +65,8 @@ class InputService : AccessibilityService() {
    private var isWheelActionsPolling = false
    private var isWaitingLongPress = false

+    private var fakeEditTextForTextStateCalculation: EditText? = null
+
    @RequiresApi(Build.VERSION_CODES.N)
    fun onMouseInput(mask: Int, _x: Int, _y: Int) {
        val x = max(0, _x)
@@ -255,20 +262,87 @@ class InputService : AccessibilityService() {
    }

    @RequiresApi(Build.VERSION_CODES.N)
-    fun onTextInput(str: String) {
-        findFocus(AccessibilityNodeInfo.FOCUS_INPUT)?.let {
-            val arguments = Bundle()
-            arguments.putCharSequence(
-                AccessibilityNodeInfo.ACTION_ARGUMENT_SET_TEXT_CHARSEQUENCE,
-                str
-            )
-            it.performAction(AccessibilityNodeInfo.ACTION_SET_TEXT, arguments)
+    fun onKeyEvent(data: ByteArray) {
+        val keyEvent = KeyEvent.parseFrom(data);
+        val handler = Handler(Looper.getMainLooper())
+        handler.post {
+            findFocus(AccessibilityNodeInfo.FOCUS_INPUT)?.let { node ->
+                val text = node.getText()
+                val isShowingHint = node.isShowingHintText()
+
+                var textSelectionStart = node.getTextSelectionStart()
+                var textSelectionEnd = node.getTextSelectionEnd()
+
+                if (text != null) {
+                    if (textSelectionStart > text.length) {
+                        textSelectionStart = text.length
+                    }
+                    if (textSelectionEnd > text.length) {
+                        textSelectionEnd = text.length
+                    }
+                    if (textSelectionStart > textSelectionEnd) {
+                        textSelectionStart = textSelectionEnd
+                    }
+                }
+
+                if (keyEvent.hasSeq()) {
+                    val seq = keyEvent.getSeq()
+
+                    var newText = ""
+
+                    if ((textSelectionStart == -1) || (textSelectionEnd == -1)) {
+                        newText = seq
+                    } else {
+                        newText = text.let {
+                            it.substring(0, textSelectionStart) + seq + it.substring(textSelectionStart)
+                        }
+                    }
+
+                    val arguments = Bundle()
+                    arguments.putCharSequence(
+                        AccessibilityNodeInfo.ACTION_ARGUMENT_SET_TEXT_CHARSEQUENCE,
+                        newText
+                    )
+                    node.performAction(AccessibilityNodeInfo.ACTION_SET_TEXT, arguments)
+
+                } else {
+                    KeyEventConverter.toAndroidKeyEvent(keyEvent).let { event ->
+                        Log.d(logTag, "event $event text $text start $textSelectionStar end $textSelectionEnd")
+                        if (isShowingHint) {
+                            this.fakeEditTextForTextStateCalculation?.setText(null)
+                        } else {
+                            this.fakeEditTextForTextStateCalculation?.setText(text)
+                        }
+                        if (textSelectionStart != -1 && textSelectionEnd != -1) {
+                            this.fakeEditTextForTextStateCalculation?.setSelection(
+                                textSelectionStart,
+                                textSelectionEnd
+                            )
+                        }
+                        this.fakeEditTextForTextStateCalculation?.dispatchKeyEvent(event)
+
+                        this.fakeEditTextForTextStateCalculation?.getText()?.let { newText ->
+                            val arguments = Bundle()
+                            arguments.putCharSequence(
+                                AccessibilityNodeInfo.ACTION_ARGUMENT_SET_TEXT_CHARSEQUENCE,
+                                newText.toString()
+                            )
+                            node.performAction(AccessibilityNodeInfo.ACTION_SET_TEXT, arguments)
+                        }
+                    }
+                }
+            }
        }
    }

+
+    override fun onAccessibilityEvent(event: AccessibilityEvent) {
+    }
+
    override fun onServiceConnected() {
        super.onServiceConnected()
        ctx = this
+        fakeEditTextForTextStateCalculation = EditText(this)
        Log.d(logTag, "onServiceConnected!")
    }

@@ -277,7 +351,5 @@ class InputService : AccessibilityService() {
        super.onDestroy()
    }

-    override fun onAccessibilityEvent(event: AccessibilityEvent?) {}
-
    override fun onInterrupt() {}
 }
--- a/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/KeyboardKeyEventMapper.kt
+++ b/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/KeyboardKeyEventMapper.kt
@@ -0,0 +1,115 @@
+package hbb;
+import android.view.KeyEvent
+import android.view.KeyCharacterMap
+import hbb.MessageOuterClass.ControlKey
+
+object KeyEventConverter {
+    fun toAndroidKeyEvent(keyEventProto: hbb.MessageOuterClass.KeyEvent): KeyEvent {
+        var chrValue = 0
+        var modifiers = 0
+
+        android.util.Log.d(tag, "proto: $keyEventProto")
+
+        if (keyEventProto.hasUnicode()) {
+            chrValue = 
+        }
+
+        if (keyEventProto.hasChr()) {
+            chrValue = convertUnicodeToKeyCode(keyEventProto.getChr() as Int)
+        } else if (keyEventProto.hasControlKey()) {
+            chrValue = convertControlKeyToKeyCode(keyEventProto.getControlKey())
+        }
+
+        var modifiersList = keyEventProto.getModifiersList()
+
+        if (modifiersList != null) {
+            for (modifier in keyEventProto.getModifiersList()) {
+                val modifierValue = convertModifier(modifier)
+                modifiers = modifiers and modifierValue
+            }
+        }
+
+        var action = 0
+        if (keyEventProto.getDown()) {
+            action = KeyEvent.ACTION_DOWN
+        } else {
+            action = KeyEvent.ACTION_UP
+        }
+
+        return KeyEvent(0, 0, action, chrValue, 0, modifiers)
+    }
+
+    private fun convertModifier(controlKey: hbb.MessageOuterClass.ControlKey): Int {
+        // Add logic to map ControlKey values to Android KeyEvent key codes.
+        // You'll need to provide the mapping for each key.
+        return when (controlKey) {
+            ControlKey.Alt -> KeyEvent.META_ALT_ON
+            ControlKey.Control -> KeyEvent.META_CTRL_ON
+            ControlKey.CapsLock -> KeyEvent.META_CAPS_LOCK_ON
+            ControlKey.Meta -> KeyEvent.META_META_ON
+            ControlKey.NumLock -> KeyEvent.META_NUM_LOCK_ON
+            ControlKey.RShift -> KeyEvent.META_SHIFT_RIGHT_ON
+            ControlKey.Shift -> KeyEvent.META_SHIFT_ON
+            ControlKey.RAlt -> KeyEvent.META_ALT_RIGHT_ON
+            ControlKey.RControl -> KeyEvent.META_CTRL_RIGHT_ON
+            else -> 0 // Default to unknown.
+        }
+    }
+
+    private val tag = "KeyEventConverter"
+
+    private fun convertUnicodeToKeyCode(unicode: Int): Int {
+        val charMap = KeyCharacterMap.load(KeyCharacterMap.VIRTUAL_KEYBOARD)
+        android.util.Log.d(tag, "unicode: $unicode")
+        val events = charMap.getEvents(charArrayOf(unicode.toChar()))
+        if (events != null && events.size > 0) {
+            android.util.Log.d(tag, "keycode ${events[0].keyCode}")
+            return events[0].keyCode
+        }
+        return 0
+    }
+
+    private fun convertControlKeyToKeyCode(controlKey: hbb.MessageOuterClass.ControlKey): Int {
+        // Add logic to map ControlKey values to Android KeyEvent key codes.
+        // You'll need to provide the mapping for each key.
+        return when (controlKey) {
+            ControlKey.Alt -> KeyEvent.KEYCODE_ALT_LEFT
+            ControlKey.Backspace -> KeyEvent.KEYCODE_DEL
+            ControlKey.Control -> KeyEvent.KEYCODE_CTRL_LEFT
+            ControlKey.CapsLock -> KeyEvent.KEYCODE_CAPS_LOCK
+            ControlKey.Meta -> KeyEvent.KEYCODE_META_LEFT
+            ControlKey.NumLock -> KeyEvent.KEYCODE_NUM_LOCK
+            ControlKey.RShift -> KeyEvent.KEYCODE_SHIFT_RIGHT
+            ControlKey.Shift -> KeyEvent.KEYCODE_SHIFT_LEFT
+            ControlKey.RAlt -> KeyEvent.KEYCODE_ALT_RIGHT
+            ControlKey.RControl -> KeyEvent.KEYCODE_CTRL_RIGHT
+            ControlKey.DownArrow -> KeyEvent.KEYCODE_DPAD_DOWN
+            ControlKey.LeftArrow -> KeyEvent.KEYCODE_DPAD_LEFT
+            ControlKey.RightArrow -> KeyEvent.KEYCODE_DPAD_RIGHT
+            ControlKey.UpArrow -> KeyEvent.KEYCODE_DPAD_UP
+            ControlKey.End -> KeyEvent.KEYCODE_MOVE_END
+            ControlKey.Home -> KeyEvent.KEYCODE_MOVE_HOME
+            ControlKey.Insert -> KeyEvent.KEYCODE_INSERT
+            ControlKey.Escape -> KeyEvent.KEYCODE_ESCAPE
+            ControlKey.F1 -> KeyEvent.KEYCODE_F1
+            ControlKey.F2 -> KeyEvent.KEYCODE_F2
+            ControlKey.F3 -> KeyEvent.KEYCODE_F3
+            ControlKey.F4 -> KeyEvent.KEYCODE_F4
+            ControlKey.F5 -> KeyEvent.KEYCODE_F5
+            ControlKey.F6 -> KeyEvent.KEYCODE_F6
+            ControlKey.F7 -> KeyEvent.KEYCODE_F7
+            ControlKey.F8 -> KeyEvent.KEYCODE_F8
+            ControlKey.F9 -> KeyEvent.KEYCODE_F9
+            ControlKey.F10 -> KeyEvent.KEYCODE_F10
+            ControlKey.F11 -> KeyEvent.KEYCODE_F11
+            ControlKey.F12 -> KeyEvent.KEYCODE_F12
+            ControlKey.Space -> KeyEvent.KEYCODE_SPACE
+            ControlKey.Tab -> KeyEvent.KEYCODE_TAB
+            ControlKey.Return -> KeyEvent.KEYCODE_ENTER
+            ControlKey.Delete -> KeyEvent.KEYCODE_FORWARD_DEL
+            ControlKey.Clear -> KeyEvent.KEYCODE_CLEAR
+            ControlKey.Pause -> KeyEvent.KEYCODE_BREAK
+            else -> 0 // Default to unknown.
+        }
+    }
+}
--- a/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/MainService.kt
+++ b/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/MainService.kt
@@ -44,7 +44,6 @@ import java.nio.ByteBuffer
 import kotlin.math.max
 import kotlin.math.min

-
 const val DEFAULT_NOTIFY_TITLE = "RustDesk"
 const val DEFAULT_NOTIFY_TEXT = "Service is running"
 const val DEFAULT_NOTIFY_ID = 1
@@ -96,8 +95,8 @@ class MainService : Service() {

    @Keep
    @RequiresApi(Build.VERSION_CODES.N)
-    fun rustInputString(input: String) {
-        InputService.ctx?.onTextInput(input)
+    fun rustKeyEventInput(input: ByteArray) {
+        InputService.ctx?.onKeyEvent(input)
    }

    @Keep
--- a/flutter/android/app/src/main/res/xml/accessibility_service_config.xml
+++ b/flutter/android/app/src/main/res/xml/accessibility_service_config.xml
@@ -1,5 +1,6 @@
 <accessibility-service xmlns:android="http://schemas.android.com/apk/res/android"
    android:accessibilityEventTypes="typeWindowsChanged"
+    android:canRetrieveWindowContent="true"
    android:accessibilityFlags="flagDefault"
    android:notificationTimeout="50"
    android:description="@string/accessibility_service_description"