Merge pull request #259 from webadderall/fix/audio-desync-cursor-telemetry

fix: audio desync and cursor telemetry under CPU load

electron/ipc/cursor/telemetry.ts

@@ -26,7 +26,7 @@ export function clamp(value: number, min: number, max: number) {
 
 export function stopCursorCapture() {
 	if (cursorCaptureInterval) {
-		clearInterval(cursorCaptureInterval);
+		clearTimeout(cursorCaptureInterval);
 		setCursorCaptureInterval(null);
 	}
 }
@@ -155,13 +155,33 @@ export function snapshotCursorTelemetryForPersistence() {
 
 export function startCursorSampling() {
 	stopCursorCapture();
-	setCursorCaptureInterval(
-		setInterval(() => {
-			if (isCursorCaptureActive) {
-				sampleCursorPoint();
-			}
-		}, CURSOR_SAMPLE_INTERVAL_MS),
-	);
+
+	// Use recursive setTimeout with drift compensation instead of setInterval.
+	// Under CPU load setInterval bunches or skips callbacks, creating large gaps
+	// in telemetry data.  This approach measures wall-clock drift each tick and
+	// adjusts the next delay so samples stay close to the target interval.
+	let nextExpectedMs = Date.now() + CURSOR_SAMPLE_INTERVAL_MS;
+
+	const tick = () => {
+		if (isCursorCaptureActive) {
+			sampleCursorPoint();
+		}
+
+		const now = Date.now();
+		const drift = now - nextExpectedMs;
+		nextExpectedMs += CURSOR_SAMPLE_INTERVAL_MS;
+
+		// If we fell behind by more than one full interval, reset the baseline
+		// so we don't try to "catch up" with a burst of rapid samples.
+		if (drift > CURSOR_SAMPLE_INTERVAL_MS) {
+			nextExpectedMs = now + CURSOR_SAMPLE_INTERVAL_MS;
+		}
+
+		const delay = Math.max(1, nextExpectedMs - now);
+		setCursorCaptureInterval(setTimeout(tick, delay));
+	};
+
+	setCursorCaptureInterval(setTimeout(tick, CURSOR_SAMPLE_INTERVAL_MS));
 }
 
 // Re-export for consumers that use it from this module

electron/ipc/ffmpeg/filters.ts

@@ -40,7 +40,7 @@ export function getAudioSyncAdjustment(
 
 	const durationDeltaMs = Math.round((videoDuration - audioDuration) * 1000);
 	const absDeltaMs = Math.abs(durationDeltaMs);
-	if (absDeltaMs <= 50) {
+	if (absDeltaMs <= 20) {
 		return { mode: "none", delayMs: 0, tempoRatio: 1, durationDeltaMs };
 	}
 

electron/ipc/recording/mac.ts

@@ -194,53 +194,35 @@ export async function muxNativeMacRecordingWithAudio(
 		tempoRatio: 1,
 		durationDeltaMs: 0,
 	};
-	const needsFilter = systemAdjustment.mode !== "none" || micAdjustment.mode !== "none";
 
+	// Always route through the filter graph so that aresample=async=1 is
+	// applied to every audio stream.  This corrects progressive clock drift
+	// between the video and audio tracks that a simple duration comparison
+	// cannot detect (e.g. audio gradually falling behind under CPU load).
 	let args: string[];
 	if (availableAudioInputs.length === 2) {
-		if (needsFilter) {
-			const filterParts: string[] = [];
-			appendSyncedAudioFilter(filterParts, "[1:a]", "s", systemAdjustment);
-			appendSyncedAudioFilter(filterParts, "[2:a]", "m", micAdjustment);
-			filterParts.push("[s][m]amix=inputs=2:duration=longest:normalize=0[aout]");
-			args = [
-				"-y",
-				...inputs,
-				"-filter_complex",
-				filterParts.join(";"),
-				"-map",
-				"0:v:0",
-				"-map",
-				"[aout]",
-				"-c:v",
-				"copy",
-				"-c:a",
-				"aac",
-				"-b:a",
-				"192k",
-				"-shortest",
-				mixedOutputPath,
-			];
-		} else {
-			args = [
-				"-y",
-				...inputs,
-				"-filter_complex",
-				"[1:a][2:a]amix=inputs=2:duration=longest:normalize=0[aout]",
-				"-map",
-				"0:v:0",
-				"-map",
-				"[aout]",
-				"-c:v",
-				"copy",
-				"-c:a",
-				"aac",
-				"-b:a",
-				"192k",
-				"-shortest",
-				mixedOutputPath,
-			];
-		}
+		const filterParts: string[] = [];
+		appendSyncedAudioFilter(filterParts, "[1:a]", "s", systemAdjustment);
+		appendSyncedAudioFilter(filterParts, "[2:a]", "m", micAdjustment);
+		filterParts.push("[s][m]amix=inputs=2:duration=longest:normalize=0[aout]");
+		args = [
+			"-y",
+			...inputs,
+			"-filter_complex",
+			filterParts.join(";"),
+			"-map",
+			"0:v:0",
+			"-map",
+			"[aout]",
+			"-c:v",
+			"copy",
+			"-c:a",
+			"aac",
+			"-b:a",
+			"192k",
+			"-shortest",
+			mixedOutputPath,
+		];
 	} else {
 		const singleAdjustment = audioAdjustments.get(availableAudioInputs[0]) ?? {
 			mode: "none",
@@ -248,45 +230,26 @@ export async function muxNativeMacRecordingWithAudio(
 			tempoRatio: 1,
 			durationDeltaMs: 0,
 		};
-		if (singleAdjustment.mode !== "none") {
-			const filterParts: string[] = [];
-			appendSyncedAudioFilter(filterParts, "[1:a]", "aout", singleAdjustment);
-			args = [
-				"-y",
-				...inputs,
-				"-filter_complex",
-				filterParts.join(";"),
-				"-map",
-				"0:v:0",
-				"-map",
-				"[aout]",
-				"-c:v",
-				"copy",
-				"-c:a",
-				"aac",
-				"-b:a",
-				"192k",
-				"-shortest",
-				mixedOutputPath,
-			];
-		} else {
-			args = [
-				"-y",
-				...inputs,
-				"-map",
-				"0:v:0",
-				"-map",
-				"1:a:0",
-				"-c:v",
-				"copy",
-				"-c:a",
-				"aac",
-				"-b:a",
-				"192k",
-				"-shortest",
-				mixedOutputPath,
-			];
-		}
+		const filterParts: string[] = [];
+		appendSyncedAudioFilter(filterParts, "[1:a]", "aout", singleAdjustment);
+		args = [
+			"-y",
+			...inputs,
+			"-filter_complex",
+			filterParts.join(";"),
+			"-map",
+			"0:v:0",
+			"-map",
+			"[aout]",
+			"-c:v",
+			"copy",
+			"-c:a",
+			"aac",
+			"-b:a",
+			"192k",
+			"-shortest",
+			mixedOutputPath,
+		];
 	}
 
 	console.log("[mux] Running ffmpeg:", ffmpegPath, args.join(" "));

electron/ipc/recording/windows.ts

@@ -278,58 +278,38 @@ export async function muxNativeWindowsVideoWithAudio(
 			durationDeltaMs: 0,
 		};
 
-		if (pauseFilter || singleAdjustment.mode !== "none") {
-			const filterParts: string[] = [];
-			if (pauseFilter) {
-				filterParts.push(pauseFilter);
-			}
-			const srcLabel = pauseFilter ? "[trimmed_audio]" : "[1:a]";
-			appendSyncedAudioFilter(filterParts, srcLabel, "aout", singleAdjustment);
-
-			await execFileAsync(
-				ffmpegPath,
-				[
-					"-y",
-					...inputs,
-					"-filter_complex",
-					filterParts.join(";"),
-					"-map",
-					"0:v:0",
-					"-map",
-					"[aout]",
-					"-c:v",
-					"copy",
-					"-c:a",
-					"aac",
-					"-b:a",
-					"192k",
-					"-shortest",
-					mixedOutputPath,
-				],
-				{ timeout: 120000, maxBuffer: 10 * 1024 * 1024 },
-			);
-		} else {
-			await execFileAsync(
-				ffmpegPath,
-				[
-					"-y",
-					...inputs,
-					"-map",
-					"0:v:0",
-					"-map",
-					"1:a:0",
-					"-c:v",
-					"copy",
-					"-c:a",
-					"aac",
-					"-b:a",
-					"192k",
-					"-shortest",
-					mixedOutputPath,
-				],
-				{ timeout: 120000, maxBuffer: 10 * 1024 * 1024 },
-			);
+		// Always route through the filter graph so that aresample=async=1 is
+		// applied.  This corrects progressive clock drift between video and
+		// audio tracks that a simple duration comparison cannot detect.
+		const filterParts: string[] = [];
+		if (pauseFilter) {
+			filterParts.push(pauseFilter);
 		}
+		const srcLabel = pauseFilter ? "[trimmed_audio]" : "[1:a]";
+		appendSyncedAudioFilter(filterParts, srcLabel, "aout", singleAdjustment);
+
+		await execFileAsync(
+			ffmpegPath,
+			[
+				"-y",
+				...inputs,
+				"-filter_complex",
+				filterParts.join(";"),
+				"-map",
+				"0:v:0",
+				"-map",
+				"[aout]",
+				"-c:v",
+				"copy",
+				"-c:a",
+				"aac",
+				"-b:a",
+				"192k",
+				"-shortest",
+				mixedOutputPath,
+			],
+			{ timeout: 120000, maxBuffer: 10 * 1024 * 1024 },
+		);
 	}
 
 	await moveFileWithOverwrite(mixedOutputPath, videoPath);

electron/ipc/register/project.ts

@@ -379,17 +379,23 @@ export function registerProjectHandlers() {
     }
   });
 
-  ipcMain.handle('get-local-media-url', (_, filePath: string) => {
+  ipcMain.handle('get-local-media-url', async (_, filePath: string) => {
     const baseUrl = getMediaServerBaseUrl();
     if (!baseUrl || !filePath) {
       return { success: false as const };
     }
-    const resolved = path.resolve(filePath);
-    if (!approvedLocalReadPaths.has(resolved)) {
+    const normalized = path.resolve(filePath);
+    let resolved: string;
+    try {
+      resolved = await fs.realpath(normalized);
+    } catch {
+      return { success: false as const };
+    }
+    if (!approvedLocalReadPaths.has(resolved) && !approvedLocalReadPaths.has(normalized)) {
       console.warn(`[get-local-media-url] Blocked unapproved path: ${resolved}`);
       return { success: false as const };
     }
-    return { success: true as const, url: buildMediaUrl(baseUrl, filePath) };
+    return { success: true as const, url: buildMediaUrl(baseUrl, resolved) };
   });
 
 }

electron/ipc/register/recording.ts

@@ -5,7 +5,7 @@ import path from "node:path";
 import { promisify } from "node:util";
 import { app, BrowserWindow, desktopCapturer, dialog, ipcMain, shell, systemPreferences } from "electron";
 import { showCursor } from "../../cursorHider";
-import { ALLOW_RECORDLY_WINDOW_CAPTURE, CURSOR_SAMPLE_INTERVAL_MS } from "../constants";
+import { ALLOW_RECORDLY_WINDOW_CAPTURE } from "../constants";
 import type { SelectedSource, NativeMacRecordingOptions, PauseSegment, CursorTelemetryPoint } from "../types";
 import {
 	selectedSource,
@@ -53,7 +53,6 @@ import {
 	setCachedSystemCursorAssets,
 	cachedSystemCursorAssetsSourceMtimeMs,
 	setCachedSystemCursorAssetsSourceMtimeMs,
-	setCursorCaptureInterval,
 	setCursorCaptureStartTimeMs,
 	setActiveCursorSamples,
 	setPendingCursorSamples,
@@ -113,6 +112,7 @@ import {
 	clamp,
 	stopCursorCapture,
 	sampleCursorPoint,
+	startCursorSampling,
 	snapshotCursorTelemetryForPersistence,
 } from "../cursor/telemetry";
 import {
@@ -1083,7 +1083,7 @@ export function registerRecordingHandlers(
       setLinuxCursorScreenPoint(null)
       setLastLeftClick(null)
       sampleCursorPoint()
-      setCursorCaptureInterval(setInterval(sampleCursorPoint, CURSOR_SAMPLE_INTERVAL_MS))
+      startCursorSampling()
       void startInteractionCapture()
     } else {
       setIsCursorCaptureActive(false)

src/components/video-editor/VideoEditor.tsx

@@ -781,7 +781,7 @@ export default function VideoEditor() {
 					padding,
 					cropRegion,
 					webcam,
-					webcamUrl: webcam.sourcePath ? toFileUrl(webcam.sourcePath) : null,
+					webcamUrl: resolvedWebcamVideoUrl ?? (webcam.sourcePath ? toFileUrl(webcam.sourcePath) : null),
 					videoWidth: previewVideo.videoWidth,
 					videoHeight: previewVideo.videoHeight,
 					annotationRegions,
@@ -1807,7 +1807,6 @@ export default function VideoEditor() {
 			setResolvedWebcamVideoUrl(null);
 			return;
 		}
-		setResolvedWebcamVideoUrl(null);
 		void resolveVideoUrl(webcam.sourcePath).then((url) => {
 			if (!cancelled) setResolvedWebcamVideoUrl(url);
 		});
@@ -3540,7 +3539,7 @@ export default function VideoEditor() {
 						videoPadding: padding,
 						cropRegion,
 						webcam,
-						webcamUrl: webcam.sourcePath ? toFileUrl(webcam.sourcePath) : null,
+						webcamUrl: resolvedWebcamVideoUrl ?? (webcam.sourcePath ? toFileUrl(webcam.sourcePath) : null),
 						annotationRegions,
 						autoCaptions,
 						autoCaptionSettings,
@@ -3709,7 +3708,7 @@ export default function VideoEditor() {
 						padding,
 						cropRegion,
 						webcam,
-						webcamUrl: webcam.sourcePath ? toFileUrl(webcam.sourcePath) : null,
+						webcamUrl: resolvedWebcamVideoUrl ?? (webcam.sourcePath ? toFileUrl(webcam.sourcePath) : null),
 						annotationRegions,
 						autoCaptions,
 						autoCaptionSettings,

src/hooks/useScreenRecorder.ts

@@ -18,7 +18,7 @@ const HIGH_FRAME_RATE_BOOST = 1.7;
 const DEFAULT_WIDTH = 1920;
 const DEFAULT_HEIGHT = 1080;
 const CODEC_ALIGNMENT = 2;
-const RECORDER_TIMESLICE_MS = 1000;
+const RECORDER_TIMESLICE_MS = 250;
 const BITS_PER_MEGABIT = 1_000_000;
 const MIN_FRAME_RATE = 30;
 const CHROME_MEDIA_SOURCE = "desktop";
@@ -968,7 +968,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn {
 									micFallbackChunks.current.push(event.data);
 								}
 							};
-							recorder.start(1000);
+							recorder.start(RECORDER_TIMESLICE_MS);
 							micFallbackRecorder.current = recorder;
 						} catch (micError) {
 							console.warn("Browser microphone fallback failed:", micError);
@@ -1090,7 +1090,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn {
 				const micAudioTrack = microphoneStream.current?.getAudioTracks()[0];
 
 				if (systemAudioTrack && micAudioTrack) {
-					const context = new AudioContext();
+					const context = new AudioContext({ sampleRate: 48000 });
 					mixingContext.current = context;
 					const systemSource = context.createMediaStreamSource(
 						new MediaStream([systemAudioTrack]),

src/lib/exporter/audioEncoder.ts

@@ -18,6 +18,12 @@ const DECODE_BACKPRESSURE_LIMIT = 20;
 const ENCODE_BACKPRESSURE_LIMIT = 20;
 const MIN_SPEED_REGION_DELTA_MS = 0.0001;
 const MP4_AUDIO_CODEC = "mp4a.40.2";
+const SYNC_SEEK_THRESHOLD_SEC = 0.15;
+const SYNC_PLAYBACK_RATE_OPTIONS = {
+	toleranceSeconds: 0.008,
+	correctionWindowSeconds: 0.5,
+	maxAdjustment: 0.12,
+} as const;
 
 export async function isAacAudioEncodingSupported(
 		sampleRate = 48_000,
@@ -555,7 +561,7 @@ export class AudioProcessor {
 				throw new Error("Export cancelled");
 			}
 
-			audioContext = new AudioContext();
+			audioContext = new AudioContext({ sampleRate: 48000 });
 			const currentDestinationNode = audioContext.createMediaStreamDestination();
 			destinationNode = currentDestinationNode;
 
@@ -721,7 +727,7 @@ export class AudioProcessor {
 							continue;
 						}
 
-						if (Math.abs(audioEl.currentTime - targetTimeSec) > 0.3) {
+						if (Math.abs(audioEl.currentTime - targetTimeSec) > SYNC_SEEK_THRESHOLD_SEC) {
 							audioEl.currentTime = targetTimeSec;
 						}
 
@@ -729,6 +735,7 @@ export class AudioProcessor {
 							basePlaybackRate: playbackRate,
 							currentTime: audioEl.currentTime,
 							targetTime: targetTimeSec,
+							...SYNC_PLAYBACK_RATE_OPTIONS,
 						});
 						if (Math.abs(audioEl.playbackRate - syncedPlaybackRate) > 0.0001) {
 							audioEl.playbackRate = syncedPlaybackRate;
@@ -748,7 +755,7 @@ export class AudioProcessor {
 
 						if (isInRegion) {
 							const audioOffset = (currentTimeMs - region.startMs) / 1000;
-							if (Math.abs(audioEl.currentTime - audioOffset) > 0.3) {
+							if (Math.abs(audioEl.currentTime - audioOffset) > SYNC_SEEK_THRESHOLD_SEC) {
 								audioEl.currentTime = audioOffset;
 							}
 
@@ -756,6 +763,7 @@ export class AudioProcessor {
 								basePlaybackRate: playbackRate,
 								currentTime: audioEl.currentTime,
 								targetTime: audioOffset,
+								...SYNC_PLAYBACK_RATE_OPTIONS,
 							});
 							if (Math.abs(audioEl.playbackRate - syncedPlaybackRate) > 0.0001) {
 								audioEl.playbackRate = syncedPlaybackRate;

src/lib/exporter/modernFrameRenderer.ts

@@ -2336,21 +2336,53 @@ export class FrameRenderer {
 			return null;
 		}
 
-		let closest = telemetry[0];
-		let minDist = Math.abs(telemetry[0].timeMs - timeMs);
-		for (let i = 1; i < telemetry.length; i++) {
-			const dist = Math.abs(telemetry[i].timeMs - timeMs);
-			if (dist < minDist) {
-				minDist = dist;
-				closest = telemetry[i];
-			}
-			if (telemetry[i].timeMs > timeMs) {
-				break;
+		// Clamp to first/last sample when out of range
+		if (timeMs <= telemetry[0].timeMs) {
+			const s = telemetry[0];
+			return mapCursorToCanvasNormalized(
+				{ cx: s.cx, cy: s.cy, interactionType: s.interactionType },
+				{
+					maskRect: this.layoutCache?.maskRect,
+					canvasWidth: this.config.width,
+					canvasHeight: this.config.height,
+				},
+			);
+		}
+		if (timeMs >= telemetry[telemetry.length - 1].timeMs) {
+			const s = telemetry[telemetry.length - 1];
+			return mapCursorToCanvasNormalized(
+				{ cx: s.cx, cy: s.cy, interactionType: s.interactionType },
+				{
+					maskRect: this.layoutCache?.maskRect,
+					canvasWidth: this.config.width,
+					canvasHeight: this.config.height,
+				},
+			);
+		}
+
+		// Binary search for surrounding samples
+		let lo = 0;
+		let hi = telemetry.length - 1;
+		while (lo < hi - 1) {
+			const mid = (lo + hi) >> 1;
+			if (telemetry[mid].timeMs <= timeMs) {
+				lo = mid;
+			} else {
+				hi = mid;
 			}
 		}
 
+		const a = telemetry[lo];
+		const b = telemetry[hi];
+		const span = b.timeMs - a.timeMs;
+
+		// Linear interpolation between samples
+		const t = span > 0 ? (timeMs - a.timeMs) / span : 0;
+		const cx = a.cx + (b.cx - a.cx) * t;
+		const cy = a.cy + (b.cy - a.cy) * t;
+
 		return mapCursorToCanvasNormalized(
-			{ cx: closest.cx, cy: closest.cy, interactionType: closest.interactionType },
+			{ cx, cy, interactionType: a.interactionType },
 			{
 				maskRect: this.layoutCache?.maskRect,
 				canvasWidth: this.config.width,

src/lib/extensions/extensionHost.ts

@@ -854,17 +854,31 @@ export class ExtensionHost {
 				const t = host._cursorTelemetry;
 				if (!t || t.length === 0) return null;
 
-				let closest = t[0];
-				let minDist = Math.abs(t[0].timeMs - timeMs);
-				for (let i = 1; i < t.length; i++) {
-					const dist = Math.abs(t[i].timeMs - timeMs);
-					if (dist < minDist) {
-						minDist = dist;
-						closest = t[i];
+				if (timeMs <= t[0].timeMs) return { ...t[0], timeMs };
+				if (timeMs >= t[t.length - 1].timeMs) return { ...t[t.length - 1], timeMs };
+
+				let lo = 0;
+				let hi = t.length - 1;
+				while (lo < hi - 1) {
+					const mid = (lo + hi) >> 1;
+					if (t[mid].timeMs <= timeMs) {
+						lo = mid;
+					} else {
+						hi = mid;
 					}
-					if (t[i].timeMs > timeMs) break;
 				}
-				return { ...closest };
+
+				const a = t[lo];
+				const b = t[hi];
+				const span = b.timeMs - a.timeMs;
+				const frac = span > 0 ? (timeMs - a.timeMs) / span : 0;
+
+				return {
+					...a,
+					cx: a.cx + (b.cx - a.cx) * frac,
+					cy: a.cy + (b.cy - a.cy) * frac,
+					timeMs,
+				};
 			},
 
 			getSmoothedCursor() {

src/lib/mediaTiming.test.ts

@@ -23,10 +23,11 @@ describe("clampMediaTimeToDuration", () => {
 describe("estimateCompanionAudioStartDelaySeconds", () => {
 	it("returns the positive tail gap when companion audio is shorter", () => {
 		expect(estimateCompanionAudioStartDelaySeconds(10, 9.6)).toBeCloseTo(0.4);
+		expect(estimateCompanionAudioStartDelaySeconds(10, 9.97)).toBeCloseTo(0.03);
 	});
 
 	it("ignores tiny or negative differences", () => {
-		expect(estimateCompanionAudioStartDelaySeconds(10, 9.97)).toBe(0);
+		expect(estimateCompanionAudioStartDelaySeconds(10, 9.99)).toBe(0);
 		expect(estimateCompanionAudioStartDelaySeconds(10, 10.5)).toBe(0);
 	});
 });

src/lib/mediaTiming.ts

@@ -19,7 +19,7 @@ export function estimateCompanionAudioStartDelaySeconds(
 	const safeAudioDuration = Math.max(0, audioDuration ?? 0);
 	const estimatedDelaySeconds = safeTimelineDuration - safeAudioDuration;
 
-	return estimatedDelaySeconds > 0.05 ? estimatedDelaySeconds : 0;
+	return estimatedDelaySeconds > 0.025 ? estimatedDelaySeconds : 0;
 }
 
 export function getMediaSyncPlaybackRate({