fix(webcam): stabilize frame sync

src/components/video-editor/VideoPlayback.tsx

@@ -169,7 +169,10 @@ import { clampFocusToStage as clampFocusToStageUtil } from "./videoPlayback/focu
 import { layoutVideoContent as layoutVideoContentUtil } from "./videoPlayback/layoutUtils";
 import { updateOverlayIndicator } from "./videoPlayback/overlayUtils";
 import { createVideoEventHandlers } from "./videoPlayback/videoEventHandlers";
-import { getWebcamMediaTargetTimeSeconds } from "./videoPlayback/webcamSync";
+import {
+	getWebcamMediaTargetTimeSeconds,
+	shouldSeekWebcamMedia,
+} from "./videoPlayback/webcamSync";
 import { findDominantRegion } from "./videoPlayback/zoomRegionUtils";
 import {
 	applyZoomTransform,
@@ -1836,12 +1839,15 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
 			}
 
 			const previousTimelineTime = lastWebcamSyncTimeRef.current;
-			const timelineJumped =
-				previousTimelineTime === null || Math.abs(targetTime - previousTimelineTime) > 0.25;
-			const driftThreshold = isPlaying ? 0.35 : 0.01;
 			if (
-				timelineJumped ||
-				Math.abs(webcamVideo.currentTime - mediaTargetTime) > driftThreshold
+				shouldSeekWebcamMedia({
+					desiredTime: mediaTargetTime,
+					isPlaying,
+					isSeeking: webcamVideo.seeking,
+					previousTimelineTime,
+					timelineTime: targetTime,
+					webcamCurrentTime: webcamVideo.currentTime,
+				})
 			) {
 				try {
 					webcamVideo.currentTime = mediaTargetTime;

src/components/video-editor/videoPlayback/webcamSync.test.ts

@@ -2,6 +2,7 @@ import { describe, expect, it } from "vitest";
 import {
 	getWebcamMediaTargetTimeSeconds,
 	getWebcamPreviewTargetTimeSeconds,
+	shouldSeekWebcamMedia,
 } from "./webcamSync";
 
 describe("getWebcamPreviewTargetTimeSeconds", () => {
@@ -57,3 +58,44 @@ describe("getWebcamMediaTargetTimeSeconds", () => {
 		).toBe(0);
 	});
 });
+
+describe("shouldSeekWebcamMedia", () => {
+	it("does not issue another corrective seek while the webcam element is already seeking", () => {
+		expect(
+			shouldSeekWebcamMedia({
+					desiredTime: 10.5,
+				isPlaying: true,
+				isSeeking: true,
+				previousTimelineTime: 10,
+					timelineTime: 10.5,
+				webcamCurrentTime: 9.8,
+			}),
+		).toBe(false);
+	});
+
+	it("seeks when playback drift grows beyond the active threshold", () => {
+		expect(
+			shouldSeekWebcamMedia({
+					desiredTime: 10.5,
+				isPlaying: true,
+				isSeeking: false,
+				previousTimelineTime: 10,
+					timelineTime: 10.5,
+				webcamCurrentTime: 9.8,
+			}),
+		).toBe(true);
+	});
+
+		it("does not seek when the clamped media target is already correct", () => {
+			expect(
+				shouldSeekWebcamMedia({
+					desiredTime: 1 / 60,
+					isPlaying: false,
+					isSeeking: false,
+					previousTimelineTime: 0,
+					timelineTime: 0,
+					webcamCurrentTime: 1 / 60,
+				}),
+			).toBe(false);
+		});
+});

src/components/video-editor/videoPlayback/webcamSync.ts

@@ -15,3 +15,29 @@ export function getWebcamMediaTargetTimeSeconds({
 }
 
 export const getWebcamPreviewTargetTimeSeconds = getWebcamMediaTargetTimeSeconds;
+
+export function shouldSeekWebcamMedia({
+	desiredTime,
+	isPlaying,
+	isSeeking,
+	previousTimelineTime,
+	timelineTime,
+	webcamCurrentTime,
+}: {
+	desiredTime: number;
+	isPlaying: boolean;
+	isSeeking: boolean;
+	previousTimelineTime: number | null;
+	timelineTime: number;
+	webcamCurrentTime: number;
+}): boolean {
+	if (isSeeking) {
+		return false;
+	}
+
+	const timelineJumped =
+		previousTimelineTime === null || Math.abs(timelineTime - previousTimelineTime) > 0.25;
+	const driftThreshold = isPlaying ? 0.35 : 0.01;
+
+	return timelineJumped || Math.abs(webcamCurrentTime - desiredTime) > driftThreshold;
+}

src/lib/exporter/frameRenderer.test.ts

@@ -439,6 +439,38 @@ describe("FrameRenderer webcam export path", () => {
 		expect((outputContext as MockContext).drawImage.mock.calls[0][0]).toBe(bubbleCanvas);
 	});
 
+	it("uses the live webcam frame when sync is correct on an offset webcam timeline", () => {
+		const renderer = createRenderer() as unknown as FrameRendererTestAccess & {
+			config: { webcam: { timeOffsetMs?: number } };
+		};
+		const outputContext = createMockContext();
+		const webcamVideo = new FakeVideoElement({
+			currentTime: 1.75,
+			readyState: 2,
+			videoWidth: 800,
+			videoHeight: 600,
+			duration: 10,
+		});
+
+		renderer.config.webcam = {
+			...renderer.config.webcam,
+			timeOffsetMs: 250,
+		};
+		renderer.webcamVideoElement = webcamVideo;
+		renderer.lastSyncedWebcamTime = 1.75;
+		renderer.currentVideoTime = 2;
+		renderer.animationState.appliedScale = 1;
+
+		renderer.drawWebcamOverlay(outputContext as unknown as CanvasRenderingContext2D, 1280, 720);
+
+		const bubbleCanvas = createdCanvases[0];
+		const cacheCanvas = createdCanvases[1];
+		expect(cacheCanvas).toBeDefined();
+		expect((cacheCanvas.context as MockContext).drawImage.mock.calls[0][0]).toBe(webcamVideo);
+		expect((bubbleCanvas.context as MockContext).drawImage.mock.calls[0][0]).toBe(cacheCanvas);
+		expect((outputContext as MockContext).drawImage.mock.calls[0][0]).toBe(bubbleCanvas);
+	});
+
 	it("reuses the webcam bubble canvas across frames", () => {
 		const renderer = createRenderer() as unknown as FrameRendererTestAccess;
 		const outputContext = createMockContext();

src/lib/exporter/frameRenderer.ts

@@ -2443,16 +2443,22 @@ export class FrameRenderer {
 		bubbleCtx.imageSmoothingEnabled = true;
 		bubbleCtx.imageSmoothingQuality = "high";
 
+		const expectedWebcamTargetTime = getWebcamMediaTargetTimeSeconds({
+			currentTime: this.currentVideoTime,
+			webcamDuration: Number.isFinite(webcamVideo?.duration) ? webcamVideo?.duration : null,
+			timeOffsetMs: webcam.timeOffsetMs,
+		});
+
 		const canRefreshCache =
 			hasLiveWebcamFrame &&
 			this.lastSyncedWebcamTime !== null &&
-			Math.abs(this.lastSyncedWebcamTime - this.currentVideoTime) <= 0.02 &&
+			Math.abs(this.lastSyncedWebcamTime - expectedWebcamTargetTime) <= 0.02 &&
 			(webcamDecodedFrame
 				? true
 				: Boolean(
 						webcamVideo &&
 							!webcamVideo.seeking &&
-							Math.abs(webcamVideo.currentTime - this.currentVideoTime) <= 0.02 &&
+							Math.abs(webcamVideo.currentTime - expectedWebcamTargetTime) <= 0.02 &&
 							webcamVideo.videoWidth > 0 &&
 							webcamVideo.videoHeight > 0,
 					));

src/lib/exporter/modernFrameRenderer.test.ts

@@ -547,14 +547,14 @@ describe("ModernFrameRenderer webcam export fallback", () => {
 			};
 			renderer.config.webcamUrl = "file:///tmp/webcam.webm";
 
-			await renderer.setupWebcamSource();
-			const syncPromise = renderer.syncWebcamFrame(1);
+				await renderer.setupWebcamSource();
+				const syncPromise = renderer.syncWebcamFrame(1);
 
 			await vi.advanceTimersByTimeAsync(5_001);
-			await expect(syncPromise).resolves.toBeUndefined();
+				await expect(syncPromise).resolves.toBeUndefined();
 
-			expect(cancelForwardFrameSourceMock).toHaveBeenCalled();
-			expect(destroyForwardFrameSourceMock).toHaveBeenCalled();
+				expect(cancelForwardFrameSourceMock).toHaveBeenCalled();
+				expect(destroyForwardFrameSourceMock).toHaveBeenCalled();
 			expect(revoke).toHaveBeenCalled();
 			expect(renderer.webcamForwardFrameSource).toBeNull();
 			expect(renderer.webcamVideoElement).toBeNull();