Merge pull request #1200 from AntimonyBullets/feature/ignore-hud-auto-zooms-v2

Ignore recording HUD and webcam overlay clicks when generating auto-zooms

electron/electron-env.d.ts

@@ -253,6 +253,9 @@ interface Window {
 		onEditorModeChanged: (callback: (inEditor: boolean) => void) => () => void;
 		hudOverlayRendererReady: () => void;
 		hudOverlaySetWebcamPreviewVisible: (visible: boolean) => void;
+		hudOverlaySetInteractiveBounds: (
+			bounds: Array<{ left: number; top: number; right: number; bottom: number }> | null,
+		) => void;
 		getHudOverlayCaptureProtection: () => Promise<{ success: boolean; enabled: boolean }>;
 		getHudOverlayMousePassthroughSupported: () => Promise<{
 			success: boolean;

electron/ipc/cursor/interaction.test.ts

@@ -11,10 +11,73 @@ vi.mock("electron", () => ({
 	},
 }));
 
+vi.mock("../../windows", () => ({
+	getHudOverlayInteractiveBounds: vi.fn(),
+	getHudOverlayWindow: vi.fn(() => ({
+		isVisible: () => true,
+		isMinimized: () => false,
+	})),
+}));
+
+vi.mock("../utils", () => ({
+	getScreen: vi.fn(),
+}));
+
+vi.mock("./telemetry", () => ({
+	getCursorCaptureElapsedMs: () => 42,
+	getNormalizedCursorPoint: () => ({ cx: 0.2, cy: 0.3 }),
+	isCursorCapturePaused: () => false,
+	pushCursorSample: vi.fn(),
+}));
+
+import { getHudOverlayInteractiveBounds } from "../../windows";
+import { setIsCursorCaptureActive, setLastLeftClick } from "../state";
+import { getScreen } from "../utils";
 import {
+	recordCursorMouseDown,
 	repairBundledUiohookBinaryForCurrentArch,
 	shouldStartGlobalInteractionHook,
 } from "./interaction";
+import { pushCursorSample } from "./telemetry";
+
+describe("recordCursorMouseDown with recording overlays", () => {
+	afterEach(() => {
+		setIsCursorCaptureActive(false);
+		setLastLeftClick(null);
+		vi.clearAllMocks();
+	});
+
+	it("skips clicks on the HUD and webcam preview and records clicks outside both", () => {
+		setIsCursorCaptureActive(true);
+		setLastLeftClick(null);
+		vi.mocked(getHudOverlayInteractiveBounds).mockReturnValue([
+			{ left: 100, top: 100, right: 400, bottom: 150 },
+			{ left: 450, top: 200, right: 550, bottom: 300 },
+		]);
+		const getCursorScreenPoint = vi
+			.fn()
+			.mockReturnValueOnce({ x: 200, y: 125 })
+			.mockReturnValueOnce({ x: 500, y: 250 })
+			.mockReturnValueOnce({ x: 425, y: 180 });
+		vi.mocked(getScreen).mockReturnValue({ getCursorScreenPoint } as ReturnType<
+			typeof getScreen
+		>);
+
+		recordCursorMouseDown(1);
+		recordCursorMouseDown(1);
+		expect(pushCursorSample).not.toHaveBeenCalled();
+
+		recordCursorMouseDown(1);
+		expect(pushCursorSample).toHaveBeenCalledExactlyOnceWith(
+			0.2,
+			0.3,
+			42,
+			"click",
+			undefined,
+			1,
+		);
+	});
+});
 
 describe("shouldStartGlobalInteractionHook", () => {
 	it("does not start the synchronous uiohook event tap on macOS", () => {

electron/ipc/cursor/interaction.ts

@@ -11,6 +11,8 @@ import {
 	setLastLeftClick,
 	setLinuxCursorScreenPoint,
 } from "../state";
+import { getScreen } from "../utils";
+import { getHudOverlayInteractiveBounds, getHudOverlayWindow } from "../../windows";
 import type {
 	CursorInteractionType,
 	HookMouseEvent,
@@ -192,6 +194,22 @@ export function recordCursorMouseDown(button: 1 | 2 | 3) {
 		return;
 	}
 
+	const hudBounds = getHudOverlayInteractiveBounds();
+	const hud = getHudOverlayWindow();
+	if (hud?.isVisible() && !hud.isMinimized() && hudBounds) {
+		const cursor = getScreen().getCursorScreenPoint();
+		const isInsideHudInteractiveElement = hudBounds.some(
+			(bounds) =>
+				cursor.x >= bounds.left &&
+				cursor.x <= bounds.right &&
+				cursor.y >= bounds.top &&
+				cursor.y <= bounds.bottom,
+		);
+		if (isInsideHudInteractiveElement) {
+			return;
+		}
+	}
+
 	const point = getNormalizedCursorPoint();
 	if (!point) {
 		return;

electron/preload.ts

@@ -228,6 +228,11 @@ contextBridge.exposeInMainWorld("electronAPI", {
 	hudOverlaySetWebcamPreviewVisible: (visible: boolean) => {
 		ipcRenderer.send("hud-overlay-set-webcam-preview-visible", visible);
 	},
+	hudOverlaySetInteractiveBounds: (
+		bounds: Array<{ left: number; top: number; right: number; bottom: number }> | null,
+	) => {
+		ipcRenderer.send("hud-overlay-set-interactive-bounds", bounds);
+	},
 	getHudOverlayCaptureProtection: () => {
 		return ipcRenderer.invoke("get-hud-overlay-capture-protection");
 	},

electron/windows.ts

@@ -56,6 +56,12 @@ let hudOverlayMouseReassertTimer: NodeJS.Timeout | null = null;
 let hudOverlayRecordingActive = false;
 let hudCaptureStarting = false;
 let hudOverlayWebcamPreviewVisible = false;
+let hudOverlayInteractiveBounds: Array<{
+	left: number;
+	top: number;
+	right: number;
+	bottom: number;
+}> | null = null;
 let countdownWindow: BrowserWindow | null = null;
 let capturePickerWindows: BrowserWindow[] = [];
 let areaRecordingBorderWindow: BrowserWindow | null = null;
@@ -490,6 +496,38 @@ ipcMain.on("hud-overlay-set-webcam-preview-visible", (_event, visible: boolean)
 	}
 });
 
+ipcMain.on(
+	"hud-overlay-set-interactive-bounds",
+	(
+		_event,
+		bounds:
+			| Array<{ left: number; top: number; right: number; bottom: number }>
+			| null
+			| undefined,
+	) => {
+		if (
+			!Array.isArray(bounds) ||
+			bounds.some(
+				(rect) =>
+					!rect ||
+					typeof rect !== "object" ||
+					![rect.left, rect.top, rect.right, rect.bottom].every(Number.isFinite) ||
+					rect.right < rect.left ||
+					rect.bottom < rect.top,
+			)
+		) {
+			hudOverlayInteractiveBounds = null;
+			return;
+		}
+
+		hudOverlayInteractiveBounds = bounds;
+	},
+);
+
+export function getHudOverlayInteractiveBounds() {
+	return hudOverlayInteractiveBounds;
+}
+
 let hudCaptureProtectionUpdates = Promise.resolve();
 ipcMain.handle("set-hud-overlay-capture-protection", (_event, enabled: boolean) => {
 	// Settings can be open in more than one window. Match each helper acknowledgement
@@ -733,6 +771,7 @@ export function createHudOverlayWindow(): BrowserWindow {
 		screen.removeListener("display-metrics-changed", handleDisplayMetricsChanged);
 		if (hudOverlayWindow === win) {
 			hudOverlayWindow = null;
+			hudOverlayInteractiveBounds = null;
 			recordingPreparationActive = false;
 			hudCaptureStarting = false;
 			hudOverlayRecordingActive = false;

src/components/launch/LaunchWindow.tsx

@@ -157,6 +157,52 @@ function LaunchWindowContent() {
 		docked: showRecordingWebcamPreview,
 	});
 
+	useEffect(() => {
+		const reportInteractiveBounds = () => {
+			const elements = [hudBarRef.current, recordingWebcamPreviewContainerRef.current].filter(
+				(element): element is HTMLDivElement => element !== null,
+			);
+
+			if (elements.length === 0) {
+				window.electronAPI?.hudOverlaySetInteractiveBounds?.(null);
+				return;
+			}
+
+			window.electronAPI?.hudOverlaySetInteractiveBounds?.(
+				elements.map((element) => {
+					const rect = element.getBoundingClientRect();
+					return {
+						left: window.screenX + rect.left,
+						top: window.screenY + rect.top,
+						right: window.screenX + rect.right,
+						bottom: window.screenY + rect.bottom,
+					};
+				}),
+			);
+		};
+
+		reportInteractiveBounds();
+		const resizeObserver = new ResizeObserver(reportInteractiveBounds);
+		if (hudBarRef.current) resizeObserver.observe(hudBarRef.current);
+		if (recordingWebcamPreviewContainerRef.current) {
+			resizeObserver.observe(recordingWebcamPreviewContainerRef.current);
+		}
+		window.addEventListener("mousemove", reportInteractiveBounds);
+		window.addEventListener("resize", reportInteractiveBounds);
+
+		return () => {
+			resizeObserver.disconnect();
+			window.removeEventListener("mousemove", reportInteractiveBounds);
+			window.removeEventListener("resize", reportInteractiveBounds);
+			window.electronAPI?.hudOverlaySetInteractiveBounds?.(null);
+		};
+	}, [
+		recordingHudOffset,
+		recordingWebcamPreviewContainerRef,
+		showRecordingWebcamPreview,
+		webcamPreview.webcamPreviewOffset,
+	]);
+
 	const { handleHudMouseEnter, handleHudMouseLeave, beginInteractiveHudAction } =
 		useLaunchHudInteractionState({
 			openId,