fix: correct terminal mouse selection and scroll coordinates (#15915)

Signed-off-by: fufesou <linlong1266@gmail.com>
This commit is contained in:
fufesou
2026-08-20 13:10:45 +08:00
committed by GitHub
parent 5679670506
commit 0a4b431ea2
4 changed files with 663 additions and 17 deletions
+2 -2
View File
@@ -5,7 +5,7 @@ import 'package:flutter_hbb/common.dart';
import 'package:flutter_hbb/desktop/widgets/tabbar_widget.dart';
import 'package:flutter_hbb/models/model.dart';
import 'package:flutter_hbb/models/terminal_model.dart';
import 'package:xterm/xterm.dart';
import 'package:flutter_hbb/models/terminal_mouse_handler.dart';
import 'terminal_connection_manager.dart';
class TerminalPage extends StatefulWidget {
@@ -197,7 +197,7 @@ class _TerminalPageState extends State<TerminalPage>
body: LayoutBuilder(
builder: (context, constraints) {
final heightPx = constraints.maxHeight;
return TerminalView(
return TerminalMouseInteraction(
_terminalModel.terminal,
controller: _terminalModel.terminalController,
focusNode: _terminalFocusNode,
@@ -0,0 +1,235 @@
import 'dart:async';
import 'package:flutter/gestures.dart';
import 'package:flutter/services.dart';
import 'package:xterm/xterm.dart';
const _cellIndexOffset = 1;
const _legacyCodeOffset = 32;
const _leftButtonCode = 0;
const _motionButtonCode = 32;
const _releaseButtonCode = 3;
const _shiftModifierCode = 4;
const _metaModifierCode = 8;
const _controlModifierCode = 16;
const _modifierCodeMask =
_shiftModifierCode | _metaModifierCode | _controlModifierCode;
const _normalCoordinateLimit = 223;
const _utfCoordinateLimit = 2015;
String encodeTerminalMouseReport(
MouseReportMode mode,
int button,
CellOffset position, {
bool release = false,
}) {
final x = position.x + _cellIndexOffset;
final y = position.y + _cellIndexOffset;
final reportedButton =
release ? _releaseButtonCode | (button & _modifierCodeMask) : button;
switch (mode) {
case MouseReportMode.normal:
case MouseReportMode.utf:
final limit = mode == MouseReportMode.normal
? _normalCoordinateLimit
: _utfCoordinateLimit;
final encodedButton =
String.fromCharCode(_legacyCodeOffset + reportedButton);
return '\x1b[M$encodedButton${_legacyCoordinate(x, limit)}'
'${_legacyCoordinate(y, limit)}';
case MouseReportMode.sgr:
final suffix = release ? 'm' : 'M';
return '\x1b[<$button;$x;$y$suffix';
case MouseReportMode.urxvt:
return '\x1b[${_legacyCodeOffset + reportedButton};$x;${y}M';
}
}
String _legacyCoordinate(int value, int limit) =>
value > limit ? '\x00' : String.fromCharCode(_legacyCodeOffset + value);
int _activeModifierCode() {
final keyboard = HardwareKeyboard.instance;
return (keyboard.isShiftPressed ? _shiftModifierCode : 0) |
(keyboard.isAltPressed ? _metaModifierCode : 0) |
(keyboard.isControlPressed ? _controlModifierCode : 0);
}
class TerminalMouseDragReporter {
int? _pointerId;
TerminalController? _controller;
late CellOffset _lastReportedPosition;
var _ownsControllerSuspension = false;
var _releasePending = false;
var _reporting = false;
bool handleDown(
PointerDownEvent event,
Terminal terminal,
TerminalViewState? terminalView,
) {
if (!_isPrimaryMouse(event) || !_reportsDrag(terminal.mouseMode)) {
return false;
}
if (terminalView == null || terminalView.widget.readOnly) return false;
final controller = terminalView.widget.controller;
if (controller == null ||
controller.suspendedPointerInputs ||
!controller.pointerInput.inputs.contains(PointerInput.tap)) {
return false;
}
cancel();
_pointerId = event.pointer;
_controller = controller;
_ownsControllerSuspension = true;
_releasePending = true;
_reporting = true;
controller.setSuspendPointerInput(true);
_clearSelection(controller);
final position = _cellAt(event, terminalView);
_lastReportedPosition = position;
terminal.textInput(
_report(terminal.mouseReportMode, position),
);
return true;
}
bool handleMove(
PointerMoveEvent event,
Terminal terminal,
TerminalViewState? terminalView,
) {
if (event.pointer != _pointerId) return false;
if (terminalView == null) {
cancel();
return true;
}
final reportsDrag = _reportsDrag(terminal.mouseMode);
if (!_isPrimaryMouse(event)) {
if (_releasePending && reportsDrag) {
_reportRelease(
terminal,
_reporting ? _cellAt(event, terminalView) : _lastReportedPosition,
);
}
cancel();
return true;
}
if (!_reporting || !reportsDrag) {
if (!reportsDrag) _releasePending = false;
_reporting = false;
// Keep ownership until the matching end event to suppress local selection.
final controller = _controller;
scheduleMicrotask(() => _clearSelection(controller));
return true;
}
final position = _cellAt(event, terminalView);
_lastReportedPosition = position;
terminal.textInput(
_report(terminal.mouseReportMode, position, motion: true),
);
final controller = _controller;
scheduleMicrotask(() => _clearSelection(controller));
return true;
}
bool handleEnd(
PointerEvent event,
Terminal terminal,
TerminalViewState? terminalView,
) {
if (event.pointer != _pointerId) return false;
if (terminalView != null &&
_releasePending &&
_reportsDrag(terminal.mouseMode)) {
_reportRelease(
terminal,
_reporting ? _cellAt(event, terminalView) : _lastReportedPosition,
);
}
_clearSelection(_controller);
final controller = _controller;
_pointerId = null;
// Keep xterm's tap recognizer suspended for this pointer event.
scheduleMicrotask(() {
if (_pointerId == null && identical(_controller, controller)) {
_clearSelection(controller);
cancel();
}
});
return true;
}
void cancel() {
final controller = _controller;
if (_ownsControllerSuspension) {
controller?.setSuspendPointerInput(false);
}
_pointerId = null;
_controller = null;
_ownsControllerSuspension = false;
_releasePending = false;
_reporting = false;
}
void updateController(TerminalController controller) {
final oldController = _controller;
if (_pointerId == null || oldController == null) {
cancel();
return;
}
if (identical(oldController, controller)) return;
if (_ownsControllerSuspension) {
oldController.setSuspendPointerInput(false);
}
final acceptsPointerInput = !controller.suspendedPointerInputs &&
controller.pointerInput.inputs.contains(PointerInput.tap);
_controller = controller;
_ownsControllerSuspension = acceptsPointerInput;
_reporting = _reporting && acceptsPointerInput;
if (_ownsControllerSuspension) controller.setSuspendPointerInput(true);
_clearSelection(controller);
}
void _reportRelease(Terminal terminal, CellOffset position) {
terminal.textInput(
_report(
terminal.mouseReportMode,
position,
release: true,
),
);
}
CellOffset _cellAt(PointerEvent event, TerminalViewState terminalView) {
final renderTerminal = terminalView.renderTerminal;
return renderTerminal.getCellOffset(
renderTerminal.globalToLocal(event.position),
);
}
bool _isPrimaryMouse(PointerEvent event) =>
event.kind == PointerDeviceKind.mouse &&
(event.buttons & kPrimaryMouseButton) == kPrimaryMouseButton;
bool _reportsDrag(MouseMode mode) =>
mode == MouseMode.upDownScrollDrag || mode == MouseMode.upDownScrollMove;
void _clearSelection(TerminalController? controller) {
if (controller == null || controller.selection == null) return;
controller.clearSelection();
}
String _report(
MouseReportMode mode,
CellOffset position, {
bool release = false,
bool motion = false,
}) {
final baseButton = motion ? _motionButtonCode : _leftButtonCode;
final button = baseButton | _activeModifierCode();
return encodeTerminalMouseReport(mode, button, position, release: release);
}
}
+271 -15
View File
@@ -1,10 +1,18 @@
import 'dart:async';
import 'package:flutter/gestures.dart';
import 'package:flutter/widgets.dart';
import 'package:xterm/xterm.dart';
import 'terminal_mouse_drag_reporter.dart';
/// xterm 4.0.0 encodes wheel buttons as 68..71; the extra bit reads as a Shift
/// modifier, so strict full-screen apps ignore the report and never scroll.
/// Upstream fix: TerminalStudio/xterm.dart#238.
class WheelButtonFixMouseHandler implements TerminalMouseHandler {
const WheelButtonFixMouseHandler();
const WheelButtonFixMouseHandler({this.positionProvider});
final CellOffset? Function()? positionProvider;
@override
String? call(TerminalMouseEvent event) {
@@ -23,20 +31,268 @@ class WheelButtonFixMouseHandler implements TerminalMouseHandler {
String _reportWheel(TerminalMouseEvent event) {
// Wheel buttons 4..7 go on the wire as 64..67, but `id` is 64 + 4..7.
final button = event.button.id - 4;
final x = event.position.x + 1;
final y = event.position.y + 1;
switch (event.state.mouseReportMode) {
case MouseReportMode.normal:
case MouseReportMode.utf:
final limit =
event.state.mouseReportMode == MouseReportMode.normal ? 223 : 2015;
final col = x > limit ? '\x00' : String.fromCharCode(32 + x);
final row = y > limit ? '\x00' : String.fromCharCode(32 + y);
return '\x1b[M${String.fromCharCode(32 + button)}$col$row';
case MouseReportMode.sgr:
return '\x1b[<$button;$x;${y}M';
case MouseReportMode.urxvt:
return '\x1b[${32 + button};$x;${y}M';
final position = positionProvider?.call() ?? event.position;
return encodeTerminalMouseReport(
event.state.mouseReportMode,
button,
position,
);
}
}
class TerminalMouseInteraction extends StatefulWidget {
const TerminalMouseInteraction(
this.terminal, {
super.key,
required this.controller,
this.focusNode,
this.backgroundOpacity = 1,
this.padding,
this.onSecondaryTapDown,
});
final Terminal terminal;
final TerminalController controller;
final FocusNode? focusNode;
final double backgroundOpacity;
final EdgeInsets? padding;
final void Function(TapDownDetails, CellOffset)? onSecondaryTapDown;
@override
State<TerminalMouseInteraction> createState() =>
_TerminalMouseInteractionState();
}
class _TerminalMouseInteractionState extends State<TerminalMouseInteraction> {
static const _selectionScrollInterval = Duration(milliseconds: 50);
static const _noScroll = 0;
static const _scrollUp = -1;
static const _scrollDown = 1;
final _terminalViewKey = GlobalKey<TerminalViewState>();
final _scrollController = ScrollController();
final _mouseDrag = TerminalMouseDragReporter();
late final WheelButtonFixMouseHandler _mouseHandler;
TerminalMouseHandler? _previousMouseHandler;
Offset? _pointerPosition;
Offset? _selectionPointer;
CellAnchor? _selectionBase;
Buffer? _selectionBuffer;
int? _selectionPointerId;
Timer? _selectionScrollTimer;
var _selectionHasScrolled = false;
var _scrollDirection = _noScroll;
TerminalViewState? get _terminalView => _terminalViewKey.currentState;
@override
void initState() {
super.initState();
_mouseHandler = WheelButtonFixMouseHandler(
positionProvider: _cellAtPointer,
);
_installMouseHandler(widget.terminal);
}
@override
void didUpdateWidget(TerminalMouseInteraction oldWidget) {
super.didUpdateWidget(oldWidget);
final terminalChanged = !identical(oldWidget.terminal, widget.terminal);
final controllerChanged =
!identical(oldWidget.controller, widget.controller);
if (!terminalChanged && !controllerChanged) return;
if (controllerChanged && !terminalChanged) {
_mouseDrag.updateController(widget.controller);
} else {
_mouseDrag.cancel();
}
_clearSelectionDrag();
if (!terminalChanged) return;
_restoreMouseHandler(oldWidget.terminal);
_installMouseHandler(widget.terminal);
}
void _installMouseHandler(Terminal terminal) {
_previousMouseHandler = terminal.mouseHandler;
terminal.mouseHandler = _mouseHandler;
}
void _restoreMouseHandler(Terminal terminal) {
if (identical(terminal.mouseHandler, _mouseHandler)) {
terminal.mouseHandler = _previousMouseHandler;
}
}
CellOffset? _cellAtPointer() {
final terminalView = _terminalView;
final pointerPosition = _pointerPosition;
if (terminalView == null || pointerPosition == null) return null;
final renderTerminal = terminalView.renderTerminal;
return renderTerminal.getCellOffset(
renderTerminal.globalToLocal(pointerPosition),
);
}
void _updatePointerPosition(PointerEvent event) =>
_pointerPosition = event.position;
void _handlePointerDown(PointerDownEvent event) {
_updatePointerPosition(event);
if (_mouseDrag.handleDown(event, widget.terminal, _terminalView)) {
_clearSelectionDrag();
return;
}
if (event.kind != PointerDeviceKind.mouse ||
(event.buttons & kPrimaryMouseButton) != kPrimaryMouseButton) {
return;
}
_clearSelectionDrag();
final terminalView = _terminalView;
if (terminalView == null) return;
final renderTerminal = terminalView.renderTerminal;
final localPosition = renderTerminal.globalToLocal(event.position);
final selectionBuffer = widget.terminal.buffer;
_selectionPointerId = event.pointer;
_selectionBase = selectionBuffer.createAnchorFromOffset(
renderTerminal.getCellOffset(localPosition),
);
_selectionBuffer = selectionBuffer;
_selectionPointer = localPosition;
}
void _handlePointerMove(PointerMoveEvent event) {
_updatePointerPosition(event);
if (_mouseDrag.handleMove(event, widget.terminal, _terminalView)) return;
if (event.pointer != _selectionPointerId) return;
if (event.kind != PointerDeviceKind.mouse ||
(event.buttons & kPrimaryMouseButton) != kPrimaryMouseButton) {
_clearSelectionDrag();
return;
}
final terminalView = _terminalView;
if (terminalView == null || _selectionBase == null) return;
final renderTerminal = terminalView.renderTerminal;
final localPosition = renderTerminal.globalToLocal(event.position);
_selectionPointer = localPosition;
_setScrollDirection(
_directionFor(localPosition, renderTerminal.paintBounds),
);
if (_selectionHasScrolled) {
scheduleMicrotask(() => _scrollSelection(scroll: false));
}
}
int _directionFor(Offset position, Rect bounds) {
if (position.dy < bounds.top) return _scrollUp;
if (position.dy >= bounds.bottom) return _scrollDown;
return _noScroll;
}
void _setScrollDirection(int direction) {
if (_scrollDirection == direction) return;
_stopAutoScroll();
_scrollDirection = direction;
if (direction == _noScroll) return;
_scrollSelection();
if (_scrollDirection != _noScroll) {
_selectionScrollTimer = Timer.periodic(
_selectionScrollInterval,
(_) => _scrollSelection(),
);
}
}
void _scrollSelection({bool scroll = true}) {
final terminalView = _terminalView;
final selectionBase = _selectionBase;
final selectionBuffer = _selectionBuffer;
final selectionPointer = _selectionPointer;
if (terminalView == null ||
selectionBase == null ||
selectionBuffer == null ||
selectionPointer == null ||
!_scrollController.hasClients) {
return;
}
if (!identical(selectionBuffer, widget.terminal.buffer) ||
!selectionBase.attached) {
_clearSelectionDrag();
return;
}
final renderTerminal = terminalView.renderTerminal;
if (scroll) {
final position = _scrollController.position;
final target =
(position.pixels + renderTerminal.lineHeight * _scrollDirection)
.clamp(position.minScrollExtent, position.maxScrollExtent)
.toDouble();
if (target == position.pixels) {
_stopAutoScroll();
} else {
position.jumpTo(target);
_selectionHasScrolled = true;
}
}
renderTerminal.selectCharacters(
renderTerminal.getOffset(selectionBase.offset),
selectionPointer,
);
}
void _handlePointerEnd(PointerEvent event) {
_updatePointerPosition(event);
if (!_mouseDrag.handleEnd(event, widget.terminal, _terminalView) &&
event.pointer != _selectionPointerId) return;
if (_selectionHasScrolled) _scrollSelection(scroll: false);
_clearSelectionDrag();
}
void _clearSelectionDrag() {
_selectionPointerId = null;
_selectionBase?.dispose();
_selectionBase = null;
_selectionBuffer = null;
_selectionPointer = null;
_selectionHasScrolled = false;
_stopAutoScroll();
}
void _stopAutoScroll() {
_selectionScrollTimer?.cancel();
_selectionScrollTimer = null;
_scrollDirection = _noScroll;
}
@override
void dispose() {
_mouseDrag.cancel();
_clearSelectionDrag();
_restoreMouseHandler(widget.terminal);
_scrollController.dispose();
super.dispose();
}
@override
Widget build(BuildContext context) {
return Listener(
onPointerDown: _handlePointerDown,
onPointerMove: _handlePointerMove,
onPointerUp: _handlePointerEnd,
onPointerHover: _updatePointerPosition,
onPointerCancel: _handlePointerEnd,
onPointerSignal: _updatePointerPosition,
onPointerPanZoomStart: _updatePointerPosition,
onPointerPanZoomUpdate: _updatePointerPosition,
onPointerPanZoomEnd: _updatePointerPosition,
child: TerminalView(
widget.terminal,
key: _terminalViewKey,
controller: widget.controller,
scrollController: _scrollController,
focusNode: widget.focusNode,
backgroundOpacity: widget.backgroundOpacity,
padding: widget.padding,
onSecondaryTapDown: widget.onSecondaryTapDown,
),
);
}
}
@@ -1,7 +1,33 @@
import 'package:flutter_hbb/models/terminal_mouse_handler.dart';
import 'package:flutter/gestures.dart';
import 'package:flutter/material.dart';
import 'package:flutter_test/flutter_test.dart';
import 'package:xterm/xterm.dart';
const _terminalSize = Size(400, 120);
Widget _terminalHarness(
Terminal terminal,
TerminalController controller,
) =>
MaterialApp(
home: Align(
alignment: Alignment.topLeft,
child: SizedBox(
width: _terminalSize.width,
height: _terminalSize.height,
child: TerminalMouseInteraction(
terminal,
controller: controller,
),
),
),
);
void _writeLines(Terminal terminal, int count) => terminal.write(
List.generate(count, (index) => 'line $index\r\n').join(),
);
void main() {
late Terminal terminal;
late List<String> output;
@@ -111,4 +137,133 @@ void main() {
isNull,
);
});
testWidgets('dragging below scrolls and extends selection', (tester) async {
final controller = TerminalController();
_writeLines(terminal, 80);
await tester.pumpWidget(_terminalHarness(terminal, controller));
final terminalView =
tester.state<TerminalViewState>(find.byType(TerminalView));
final scrollController = terminalView.widget.scrollController!;
scrollController.jumpTo(0);
await tester.pump();
final renderTerminal = terminalView.renderTerminal;
const localStart = Offset(20, 20);
final startCell = renderTerminal.getCellOffset(localStart);
final mouse = TestPointer(1, PointerDeviceKind.mouse);
final outside = Offset(20, renderTerminal.size.height);
await tester.handlePointerEventRecord([
PointerEventRecord(Duration.zero, [
mouse.down(renderTerminal.localToGlobal(localStart)),
mouse.move(renderTerminal.localToGlobal(outside)),
]),
PointerEventRecord(const Duration(milliseconds: 150), [
mouse.move(
renderTerminal.localToGlobal(outside + const Offset(1, 1)),
),
mouse.up(),
]),
]);
expect(scrollController.offset, greaterThan(0));
expect(controller.selection!.begin, startCell);
expect(controller.selection!.end.y, greaterThan(startCell.y));
final releasedOffset = scrollController.offset;
await tester.pump(const Duration(milliseconds: 100));
expect(scrollController.offset, releasedOffset);
});
testWidgets('tmux mouse input is reported without local selection',
(tester) async {
final controller = TerminalController();
terminal.write('\x1b[?1049h\x1b[?1002h\x1b[?1006hword');
await tester.pumpWidget(_terminalHarness(terminal, controller));
final renderTerminal = tester
.state<TerminalViewState>(find.byType(TerminalView))
.renderTerminal;
const wheel = Offset(120, 40);
await tester.sendEventToBinding(
PointerScrollEvent(
position: renderTerminal.localToGlobal(wheel),
scrollDelta: const Offset(0, 40),
),
);
await tester.pump();
final wheelCell = renderTerminal.getCellOffset(wheel);
expect(output.first, '\x1b[<65;${wheelCell.x + 1};${wheelCell.y + 1}M');
output.clear();
final clickPosition =
renderTerminal.getOffset(const CellOffset(0, 0)) + const Offset(1, 1);
final clickCell = renderTerminal.getCellOffset(clickPosition);
final mouse = await tester.createGesture(kind: PointerDeviceKind.mouse);
await mouse.down(renderTerminal.localToGlobal(clickPosition));
await mouse.up();
await tester.pump();
await mouse.down(renderTerminal.localToGlobal(clickPosition));
await mouse.up();
await tester.pump(kDoubleTapTimeout);
expect(output, [
'\x1b[<0;${clickCell.x + 1};${clickCell.y + 1}M',
'\x1b[<0;${clickCell.x + 1};${clickCell.y + 1}m',
'\x1b[<0;${clickCell.x + 1};${clickCell.y + 1}M',
'\x1b[<0;${clickCell.x + 1};${clickCell.y + 1}m',
]);
expect(controller.selection, isNull);
expect(controller.suspendedPointerInputs, isFalse);
output.clear();
const start = Offset(40, 40);
const end = Offset(240, 80);
final startCell = renderTerminal.getCellOffset(start);
final endCell = renderTerminal.getCellOffset(end);
await mouse.down(renderTerminal.localToGlobal(start));
await mouse.moveTo(renderTerminal.localToGlobal(end));
await tester.pump();
expect(output, [
'\x1b[<0;${startCell.x + 1};${startCell.y + 1}M',
'\x1b[<32;${endCell.x + 1};${endCell.y + 1}M',
]);
expect(controller.selection, isNull);
await mouse.up();
expect(output.last, '\x1b[<0;${endCell.x + 1};${endCell.y + 1}m');
expect(controller.suspendedPointerInputs, isFalse);
});
testWidgets('tmux drag stays suppressed after mouse mode is disabled',
(tester) async {
final controller = TerminalController();
terminal.write('\x1b[?1049h\x1b[?1002h\x1b[?1006hword');
await tester.pumpWidget(_terminalHarness(terminal, controller));
final renderTerminal = tester
.state<TerminalViewState>(find.byType(TerminalView))
.renderTerminal;
final mouse = await tester.createGesture(kind: PointerDeviceKind.mouse);
final start = renderTerminal.localToGlobal(const Offset(40, 40));
final end = renderTerminal.localToGlobal(const Offset(240, 80));
await mouse.down(start);
output.clear();
terminal.write('\x1b[?1002l');
await mouse.moveTo(end);
expect(output, isEmpty);
expect(controller.selection, isNull);
expect(controller.suspendedPointerInputs, isTrue);
terminal.write('\x1b[?1002h');
await mouse.moveTo(start);
expect(output, isEmpty);
expect(controller.selection, isNull);
await mouse.up();
expect(output, isEmpty);
expect(controller.suspendedPointerInputs, isFalse);
await mouse.down(start);
output.clear();
terminal.write('\x1b[?1002l');
await mouse.up();
await tester.pump(kDoubleTapTimeout);
expect(output, isEmpty);
expect(controller.selection, isNull);
expect(controller.suspendedPointerInputs, isFalse);
});
}