lee/Recordly
1
0
Fork 0
mirror of https://github.com/webadderallorg/Recordly.git synced 2026-09-24 23:05:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: add writable cursor telemetry API

Browse Source
This commit is contained in:
webadderall
2026-05-04 17:28:10 +10:00
parent 572000bf0d
commit 6cc83b31e9
5 changed files with 222 additions and 83 deletions
Download Patch File Download Diff File Expand all files Collapse all files
electron/electron-env.d.ts
Vendored
+31 -9
View File
@@ -151,12 +151,12 @@ interface Window {
message?: string;
error?: string;
}>;
pauseCursorCapture: (boundaryMs?: number) => Promise<{
pauseCursorCapture: () => Promise<{
success: boolean;
message?: string;
error?: string;
}>;
resumeCursorCapture: (boundaryMs?: number) => Promise<{
resumeCursorCapture: () => Promise<{
success: boolean;
message?: string;
error?: string;
@@ -313,6 +313,15 @@ interface Window {
message?: string;
error?: string;
}>;
setCursorTelemetry: (
videoPath: string | undefined,
samples: CursorTelemetryPoint[],
) => Promise<{
success: boolean;
samples: CursorTelemetryPoint[];
message?: string;
error?: string;
}>;
getSystemCursorAssets: () => Promise<{
success: boolean;
cursors: Record<string, SystemCursorAsset>;
@@ -410,16 +419,28 @@ interface Window {
}>;
setCurrentVideoPath: (
path: string,
options?: { preserveProjectPath?: boolean },
options?: {
preserveProjectPath?: boolean;
hideOverlayCursorByDefault?: boolean;
},
) => Promise<{ success: boolean; webcamPath: string | null }>;
setCurrentRecordingSession: (session: {
videoPath: string;
webcamPath?: string | null;
timeOffsetMs?: number;
}, options?: { preserveProjectPath?: boolean }) => Promise<{ success: boolean }>;
setCurrentRecordingSession: (
session: {
videoPath: string;
webcamPath?: string | null;
timeOffsetMs?: number;
hideOverlayCursorByDefault?: boolean;
},
options?: { preserveProjectPath?: boolean },
) => Promise<{ success: boolean }>;
getCurrentRecordingSession: () => Promise<{
success: boolean;
session?: { videoPath: string; webcamPath?: string | null; timeOffsetMs?: number };
session?: {
videoPath: string;
webcamPath?: string | null;
timeOffsetMs?: number;
hideOverlayCursorByDefault?: boolean;
};
}>;
getCurrentVideoPath: () => Promise<{ success: boolean; path?: string }>;
clearCurrentVideoPath: () => Promise<{ success: boolean }>;
@@ -529,6 +550,7 @@ interface Window {
onMenuSaveProject: (callback: () => void) => () => void;
onMenuSaveProjectAs: (callback: () => void) => () => void;
getPlatform: () => Promise<string>;
getLinuxWindowSystem: () => Promise<"wayland" | "x11" | null>;
revealInFolder: (
filePath: string,
) => Promise<{ success: boolean; error?: string; message?: string }>;
electron/ipc/cursor/telemetry.test.ts
+54
View File
@@ -1,4 +1,17 @@
import { beforeEach, describe, expect, it, vi } from "vitest";
import { CURSOR_TELEMETRY_VERSION } from "../constants";
const { writeFile, rm } = vi.hoisted(() => ({
writeFile: vi.fn(),
rm: vi.fn(),
}));
vi.mock("node:fs/promises", () => ({
default: {
writeFile,
rm,
},
}));
vi.mock("electron", () => ({
app: {
@@ -18,14 +31,18 @@ vi.mock("../utils", () => ({
import {
getCursorCaptureElapsedMs,
normalizeCursorTelemetrySamples,
pauseCursorCapture,
resetCursorCaptureClock,
resumeCursorCapture,
writeCursorTelemetry,
} from "./telemetry";
import { setCursorCaptureStartTimeMs } from "../state";
describe("cursor telemetry pause clock", () => {
beforeEach(() => {
writeFile.mockReset();
rm.mockReset();
setCursorCaptureStartTimeMs(1_000);
resetCursorCaptureClock();
});
@@ -48,4 +65,41 @@ describe("cursor telemetry pause clock", () => {
expect(getCursorCaptureElapsedMs(1_900)).toBe(550);
});
it("normalizes cursor telemetry samples before persisting them", async () => {
const samples = normalizeCursorTelemetrySamples([
{ timeMs: 30, cx: 2, cy: -1, interactionType: "click", cursorType: "pointer" },
{ timeMs: -10, cx: Number.NaN, cy: 0.2, interactionType: "drag", cursorType: "ibeam" },
{ timeMs: 10, cx: 0.25, cy: 0.75, interactionType: "move", cursorType: "text" },
]);
expect(samples).toEqual([
{ timeMs: 0, cx: 0.5, cy: 0.2, interactionType: undefined, cursorType: undefined },
{ timeMs: 10, cx: 0.25, cy: 0.75, interactionType: "move", cursorType: "text" },
{ timeMs: 30, cx: 1, cy: 0, interactionType: "click", cursorType: "pointer" },
]);
await writeCursorTelemetry("/tmp/recording.mp4", samples);
expect(writeFile).toHaveBeenCalledWith(
"/tmp/recording.cursor.json",
JSON.stringify(
{
version: CURSOR_TELEMETRY_VERSION,
samples,
},
null,
2,
),
"utf-8",
);
expect(rm).not.toHaveBeenCalled();
});
it("removes the sidecar when saving an empty cursor telemetry payload", async () => {
await writeCursorTelemetry("/tmp/recording.mp4", []);
expect(rm).toHaveBeenCalledWith("/tmp/recording.cursor.json", { force: true });
expect(writeFile).not.toHaveBeenCalled();
});
});
electron/ipc/cursor/telemetry.ts
+74 -2
View File
@@ -28,6 +28,78 @@ export function clamp(value: number, min: number, max: number) {
return Math.min(max, Math.max(min, value));
}
export function normalizeCursorTelemetrySamples(rawSamples: unknown): CursorTelemetryPoint[] {
const samples = Array.isArray(rawSamples)
? rawSamples
: Array.isArray((rawSamples as { samples?: unknown[] } | null | undefined)?.samples)
? ((rawSamples as { samples: unknown[] }).samples ?? [])
: [];
return samples
.filter((sample: unknown) => Boolean(sample && typeof sample === "object"))
.map((sample: unknown) => {
const point = sample as Partial<CursorTelemetryPoint>;
return {
timeMs:
typeof point.timeMs === "number" && Number.isFinite(point.timeMs)
? Math.max(0, point.timeMs)
: 0,
cx:
typeof point.cx === "number" && Number.isFinite(point.cx)
? clamp(point.cx, 0, 1)
: 0.5,
cy:
typeof point.cy === "number" && Number.isFinite(point.cy)
? clamp(point.cy, 0, 1)
: 0.5,
interactionType:
point.interactionType === "click" ||
point.interactionType === "double-click" ||
point.interactionType === "right-click" ||
point.interactionType === "middle-click" ||
point.interactionType === "move" ||
point.interactionType === "mouseup"
? point.interactionType
: undefined,
cursorType:
point.cursorType === "arrow" ||
point.cursorType === "text" ||
point.cursorType === "pointer" ||
point.cursorType === "crosshair" ||
point.cursorType === "open-hand" ||
point.cursorType === "closed-hand" ||
point.cursorType === "resize-ew" ||
point.cursorType === "resize-ns" ||
point.cursorType === "not-allowed"
? point.cursorType
: undefined,
};
})
.sort((a, b) => a.timeMs - b.timeMs);
}
export async function writeCursorTelemetry(videoPath: string, samples: unknown) {
const telemetryPath = getTelemetryPathForVideo(videoPath);
const normalizedSamples = normalizeCursorTelemetrySamples(samples);
if (normalizedSamples.length === 0) {
await fs.rm(telemetryPath, { force: true });
return normalizedSamples;
}
await fs.writeFile(
telemetryPath,
JSON.stringify(
{ version: CURSOR_TELEMETRY_VERSION, samples: normalizedSamples },
null,
2,
),
"utf-8",
);
return normalizedSamples;
}
export function stopCursorCapture() {
if (cursorCaptureInterval) {
clearTimeout(cursorCaptureInterval);
@@ -168,9 +240,9 @@ export function pushCursorSample(
}
}
export function sampleCursorPoint(sampledAtMs = Date.now()) {
export function sampleCursorPoint() {
const point = getNormalizedCursorPoint();
pushCursorSample(point.cx, point.cy, getCursorCaptureElapsedMs(sampledAtMs), "move");
pushCursorSample(point.cx, point.cy, getCursorCaptureElapsedMs(), "move");
}
export async function persistPendingCursorTelemetry(videoPath: string) {
electron/ipc/register/recording.ts
+37 -62
View File
@@ -18,7 +18,7 @@ import { startWindowBoundsCapture, stopWindowBoundsCapture } from "../cursor/bou
import { startInteractionCapture, stopInteractionCapture } from "../cursor/interaction";
import { startNativeCursorMonitor, stopNativeCursorMonitor } from "../cursor/monitor";
import {
clamp,
normalizeCursorTelemetrySamples,
pauseCursorCapture,
resumeCursorCapture,
resetCursorCaptureClock,
@@ -26,6 +26,7 @@ import {
snapshotCursorTelemetryForPersistence,
startCursorSampling,
stopCursorCapture,
writeCursorTelemetry,
} from "../cursor/telemetry";
import { getFfmpegBinaryPath } from "../ffmpeg/binary";
import {
@@ -1312,23 +1313,15 @@ export function registerRecordingHandlers(
}
});
ipcMain.handle("pause-cursor-capture", (_event, boundaryMs?: number) => {
const timestamp =
typeof boundaryMs === "number" && Number.isFinite(boundaryMs)
? boundaryMs
: Date.now();
sampleCursorPoint(timestamp);
pauseCursorCapture(timestamp);
ipcMain.handle("pause-cursor-capture", () => {
sampleCursorPoint();
pauseCursorCapture(Date.now());
return { success: true };
});
ipcMain.handle("resume-cursor-capture", (_event, boundaryMs?: number) => {
const timestamp =
typeof boundaryMs === "number" && Number.isFinite(boundaryMs)
? boundaryMs
: Date.now();
resumeCursorCapture(timestamp);
sampleCursorPoint(timestamp);
ipcMain.handle("resume-cursor-capture", () => {
resumeCursorCapture(Date.now());
sampleCursorPoint();
return { success: true };
});
@@ -1342,53 +1335,7 @@ export function registerRecordingHandlers(
try {
const content = await fs.readFile(telemetryPath, "utf-8");
const parsed = JSON.parse(content);
const rawSamples = Array.isArray(parsed)
? parsed
: Array.isArray(parsed?.samples)
? parsed.samples
: [];
const samples: CursorTelemetryPoint[] = rawSamples
.filter((sample: unknown) => Boolean(sample && typeof sample === "object"))
.map((sample: unknown) => {
const point = sample as Partial<CursorTelemetryPoint>;
return {
timeMs:
typeof point.timeMs === "number" && Number.isFinite(point.timeMs)
? Math.max(0, point.timeMs)
: 0,
cx:
typeof point.cx === "number" && Number.isFinite(point.cx)
? clamp(point.cx, 0, 1)
: 0.5,
cy:
typeof point.cy === "number" && Number.isFinite(point.cy)
? clamp(point.cy, 0, 1)
: 0.5,
interactionType:
point.interactionType === "click" ||
point.interactionType === "double-click" ||
point.interactionType === "right-click" ||
point.interactionType === "middle-click" ||
point.interactionType === "move" ||
point.interactionType === "mouseup"
? point.interactionType
: undefined,
cursorType:
point.cursorType === "arrow" ||
point.cursorType === "text" ||
point.cursorType === "pointer" ||
point.cursorType === "crosshair" ||
point.cursorType === "open-hand" ||
point.cursorType === "closed-hand" ||
point.cursorType === "resize-ew" ||
point.cursorType === "resize-ns" ||
point.cursorType === "not-allowed"
? point.cursorType
: undefined,
};
})
.sort((a: CursorTelemetryPoint, b: CursorTelemetryPoint) => a.timeMs - b.timeMs);
const samples = normalizeCursorTelemetrySamples(parsed);
return { success: true, samples };
} catch (error) {
@@ -1405,4 +1352,32 @@ export function registerRecordingHandlers(
};
}
});
ipcMain.handle(
"set-cursor-telemetry",
async (_, videoPath: string | undefined, samples: CursorTelemetryPoint[]) => {
const targetVideoPath = normalizeVideoSourcePath(videoPath ?? currentVideoPath);
if (!targetVideoPath) {
return {
success: false,
samples: [],
message: "No video path available for cursor telemetry",
error: "Missing video path",
};
}
try {
const normalizedSamples = await writeCursorTelemetry(targetVideoPath, samples);
return { success: true, samples: normalizedSamples };
} catch (error) {
console.error("Failed to save cursor telemetry:", error);
return {
success: false,
samples: [],
message: "Failed to save cursor telemetry",
error: String(error),
};
}
},
);
}
electron/preload.ts
+26 -10
View File
@@ -293,11 +293,11 @@ contextBridge.exposeInMainWorld("electronAPI", {
resumeNativeScreenRecording: () => {
return ipcRenderer.invoke("resume-native-screen-recording");
},
pauseCursorCapture: (boundaryMs?: number) => {
return ipcRenderer.invoke("pause-cursor-capture", boundaryMs);
pauseCursorCapture: () => {
return ipcRenderer.invoke("pause-cursor-capture");
},
resumeCursorCapture: (boundaryMs?: number) => {
return ipcRenderer.invoke("resume-cursor-capture", boundaryMs);
resumeCursorCapture: () => {
return ipcRenderer.invoke("resume-cursor-capture");
},
startFfmpegRecording: (source: ProcessedDesktopSource) => {
return ipcRenderer.invoke("start-ffmpeg-recording", source);
@@ -327,6 +327,9 @@ contextBridge.exposeInMainWorld("electronAPI", {
getCursorTelemetry: (videoPath?: string) => {
return ipcRenderer.invoke("get-cursor-telemetry", videoPath);
},
setCursorTelemetry: (videoPath: string | undefined, samples: CursorTelemetryPoint[]) => {
return ipcRenderer.invoke("set-cursor-telemetry", videoPath, samples);
},
getSystemCursorAssets: () => {
return ipcRenderer.invoke("get-system-cursor-assets");
},
@@ -436,14 +439,24 @@ contextBridge.exposeInMainWorld("electronAPI", {
}) => {
return ipcRenderer.invoke("generate-auto-captions", options);
},
setCurrentVideoPath: (path: string, options?: { preserveProjectPath?: boolean }) => {
setCurrentVideoPath: (
path: string,
options?: {
preserveProjectPath?: boolean;
hideOverlayCursorByDefault?: boolean;
},
) => {
return ipcRenderer.invoke("set-current-video-path", path, options);
},
setCurrentRecordingSession: (session: {
videoPath: string;
webcamPath?: string | null;
timeOffsetMs?: number;
}, options?: { preserveProjectPath?: boolean }) => {
setCurrentRecordingSession: (
session: {
videoPath: string;
webcamPath?: string | null;
timeOffsetMs?: number;
hideOverlayCursorByDefault?: boolean;
},
options?: { preserveProjectPath?: boolean },
) => {
return ipcRenderer.invoke("set-current-recording-session", session, options);
},
getCurrentRecordingSession: () => {
@@ -603,6 +616,9 @@ contextBridge.exposeInMainWorld("electronAPI", {
getPlatform: () => {
return ipcRenderer.invoke("get-platform");
},
getLinuxWindowSystem: () => {
return ipcRenderer.invoke("get-linux-window-system");
},
revealInFolder: (filePath: string) => {
return ipcRenderer.invoke("reveal-in-folder", filePath);
},