fix(export): preserve VFR frame presentation timing

src/lib/exporter/forwardFrameSource.ts

@@ -1,7 +1,10 @@
 import { WebDemuxer } from "web-demuxer";
 import { getEffectiveVideoStreamDurationSeconds } from "@/lib/mediaTiming";
-import { getDecodedFrameTimelineOffsetUs } from "./streamingDecoder";
 import { getLocalFilePath } from "./localMediaSource";
+import {
+	getDecodedFrameTimelineOffsetUs,
+	shouldKeepHeldFrameForTargetTime,
+} from "./streamingDecoder";
 
 const DEFAULT_MAX_DECODE_QUEUE = 12;
 const DEFAULT_MAX_PENDING_FRAMES = 32;
@@ -20,7 +23,7 @@ export interface ForwardFrameSourceMetadata {
  * Forward-only decoded frame source for monotonic timestamp access.
  *
  * This avoids per-frame HTMLVideoElement seeking during export and returns
- * the nearest decoded frame for increasing target timestamps.
+ * the frame that would still be presented at the requested timestamp.
  */
 export class ForwardFrameSource {
 	private demuxer: WebDemuxer | null = null;
@@ -312,8 +315,7 @@ export class ForwardFrameSource {
 						1_000_000,
 				),
 			);
-			const handoffBoundarySec = (this.heldFrameSec + nextFrameSec) / 2;
-			if (clampedTargetTime <= handoffBoundarySec) {
+			if (shouldKeepHeldFrameForTargetTime(clampedTargetTime, nextFrameSec)) {
 				this.pendingFrames.unshift(nextFrame);
 				return new VideoFrame(this.heldFrame, {
 					timestamp: this.heldFrame.timestamp,

src/lib/exporter/streamingDecoder.test.ts

@@ -3,8 +3,16 @@ import {
 	getDecodedFrameStartupOffsetUs,
 	getDecodedFrameTimelineOffsetUs,
 	StreamingVideoDecoder,
+	shouldKeepHeldFrameForTargetTime,
 } from "./streamingDecoder";
 
+type TestElectronApiWindow = Window &
+	typeof globalThis & {
+		electronAPI: {
+			readLocalFile: ReturnType<typeof vi.fn>;
+		};
+	};
+
 describe("StreamingVideoDecoder local media loading", () => {
 	beforeEach(() => {
 		vi.restoreAllMocks();
@@ -22,12 +30,16 @@ describe("StreamingVideoDecoder local media loading", () => {
 			success: true,
 			data: new Uint8Array([1, 2, 3]),
 		}));
-		(window as any).electronAPI.readLocalFile = readLocalFile;
+		const testWindow = globalThis.window as unknown as TestElectronApiWindow;
+		testWindow.electronAPI.readLocalFile = readLocalFile;
 		const fetchSpy = vi.fn();
 		vi.stubGlobal("fetch", fetchSpy);
 
 		const decoder = new StreamingVideoDecoder();
-		const file = await (decoder as any).loadVideoFile(
+		const decoderHarness = decoder as StreamingVideoDecoder & {
+			loadVideoFile(resourceUrl: string): Promise<File>;
+		};
+		const file = await decoderHarness.loadVideoFile(
 			"http://127.0.0.1:43123/video?path=%2Ftmp%2Fcapture.mp4",
 		);
 
@@ -92,3 +104,13 @@ describe("getDecodedFrameTimelineOffsetUs", () => {
 		).toBe(150_000);
 	});
 });
+
+describe("shouldKeepHeldFrameForTargetTime", () => {
+	it("keeps a sparse VFR frame visible until the next frame timestamp", () => {
+		expect(shouldKeepHeldFrameForTargetTime(0.75, 1)).toBe(true);
+	});
+
+	it("switches to the next frame once its presentation timestamp is reached", () => {
+		expect(shouldKeepHeldFrameForTargetTime(1, 1)).toBe(false);
+	});
+});

src/lib/exporter/streamingDecoder.ts

@@ -54,6 +54,19 @@ export function getDecodedFrameTimelineOffsetUs(
 	);
 }
 
+/**
+ * Video frames are presented from their own timestamp until the next frame starts.
+ * Using midpoint handoff works poorly for sparse VFR screen recordings because it
+ * makes visual changes appear early relative to cursor and click telemetry.
+ */
+export function shouldKeepHeldFrameForTargetTime(
+	targetTimeSec: number,
+	nextFrameStartSec: number,
+	epsilonSec = 0.001,
+): boolean {
+	return targetTimeSec < nextFrameStartSec - epsilonSec;
+}
+
 /**
  * Decodes video frames via web-demuxer + VideoDecoder in a single forward pass.
  * Way faster than seeking an HTMLVideoElement per frame.
@@ -470,8 +483,10 @@ export class StreamingVideoDecoder {
 				continue;
 			}
 
-			// Any target timestamp before this midpoint is closer to heldFrame than current frame.
-			const handoffBoundarySec = (heldFrameSec + frameTimeSec) / 2;
+			// Preserve VFR presentation timing: a decoded frame stays visible until the
+			// timestamp of the next decoded frame instead of switching at the midpoint.
+			// Midpoint handoff makes sparse window-capture recordings look like the video
+			// content updates before the matching cursor/click telemetry is overlaid.
 			while (!this.cancelled) {
 				const segmentFrameCount = segmentOutputFrameCounts[segmentIdx];
 				if (segmentFrameIndex >= segmentFrameCount) {
@@ -485,7 +500,7 @@ export class StreamingVideoDecoder {
 				if (sourceTimeSec >= currentSegment.endSec - epsilonSec) {
 					break;
 				}
-				if (sourceTimeSec > handoffBoundarySec) {
+				if (!shouldKeepHeldFrameForTargetTime(sourceTimeSec, frameTimeSec, epsilonSec)) {
 					break;
 				}