From f024c89890ecd9063439fca8bb00a6fb7e3796dc Mon Sep 17 00:00:00 2001 From: webadderall <131426131+webadderall@users.noreply.github.com> Date: Sat, 18 Apr 2026 17:58:54 +1000 Subject: [PATCH] fix: always apply aresample sync filter and tighten audio sync tolerances MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Always route audio through aresample=async=1:first_pts=0 filter during muxing on both macOS and Windows, even when duration delta is small. Previously skipped when delta ≤50ms, which left progressive clock drift (from CPU load) completely uncorrected. - Lower sync detection threshold from 50ms to 20ms so tempo correction kicks in earlier. - Use explicit 48kHz AudioContext sample rate in both browser recording and export rendering to prevent sample rate mismatch drift. - Reduce MediaRecorder timeslice from 1000ms to 250ms to reduce chunk loss under CPU pressure. - Tighten export audio sync: seek threshold 300ms→150ms, tolerance 15ms→8ms, correction window 2s→0.5s, max adjustment ±8%→±12%. - Lower companion audio start delay threshold from 50ms to 25ms. --- electron/ipc/ffmpeg/filters.ts | 2 +- electron/ipc/recording/mac.ts | 129 +++++++++++------------------- electron/ipc/recording/windows.ts | 82 +++++++------------ src/hooks/useScreenRecorder.ts | 4 +- src/lib/exporter/audioEncoder.ts | 12 ++- src/lib/mediaTiming.test.ts | 3 +- src/lib/mediaTiming.ts | 2 +- 7 files changed, 92 insertions(+), 142 deletions(-) diff --git a/electron/ipc/ffmpeg/filters.ts b/electron/ipc/ffmpeg/filters.ts index dccbd8b4..a10212ba 100644 --- a/electron/ipc/ffmpeg/filters.ts +++ b/electron/ipc/ffmpeg/filters.ts @@ -40,7 +40,7 @@ export function getAudioSyncAdjustment( const durationDeltaMs = Math.round((videoDuration - audioDuration) * 1000); const absDeltaMs = Math.abs(durationDeltaMs); - if (absDeltaMs <= 50) { + if (absDeltaMs <= 20) { return { mode: "none", delayMs: 0, tempoRatio: 1, durationDeltaMs }; } diff --git a/electron/ipc/recording/mac.ts b/electron/ipc/recording/mac.ts index d340b8e1..0d70c5be 100644 --- a/electron/ipc/recording/mac.ts +++ b/electron/ipc/recording/mac.ts @@ -194,53 +194,35 @@ export async function muxNativeMacRecordingWithAudio( tempoRatio: 1, durationDeltaMs: 0, }; - const needsFilter = systemAdjustment.mode !== "none" || micAdjustment.mode !== "none"; + // Always route through the filter graph so that aresample=async=1 is + // applied to every audio stream. This corrects progressive clock drift + // between the video and audio tracks that a simple duration comparison + // cannot detect (e.g. audio gradually falling behind under CPU load). let args: string[]; if (availableAudioInputs.length === 2) { - if (needsFilter) { - const filterParts: string[] = []; - appendSyncedAudioFilter(filterParts, "[1:a]", "s", systemAdjustment); - appendSyncedAudioFilter(filterParts, "[2:a]", "m", micAdjustment); - filterParts.push("[s][m]amix=inputs=2:duration=longest:normalize=0[aout]"); - args = [ - "-y", - ...inputs, - "-filter_complex", - filterParts.join(";"), - "-map", - "0:v:0", - "-map", - "[aout]", - "-c:v", - "copy", - "-c:a", - "aac", - "-b:a", - "192k", - "-shortest", - mixedOutputPath, - ]; - } else { - args = [ - "-y", - ...inputs, - "-filter_complex", - "[1:a][2:a]amix=inputs=2:duration=longest:normalize=0[aout]", - "-map", - "0:v:0", - "-map", - "[aout]", - "-c:v", - "copy", - "-c:a", - "aac", - "-b:a", - "192k", - "-shortest", - mixedOutputPath, - ]; - } + const filterParts: string[] = []; + appendSyncedAudioFilter(filterParts, "[1:a]", "s", systemAdjustment); + appendSyncedAudioFilter(filterParts, "[2:a]", "m", micAdjustment); + filterParts.push("[s][m]amix=inputs=2:duration=longest:normalize=0[aout]"); + args = [ + "-y", + ...inputs, + "-filter_complex", + filterParts.join(";"), + "-map", + "0:v:0", + "-map", + "[aout]", + "-c:v", + "copy", + "-c:a", + "aac", + "-b:a", + "192k", + "-shortest", + mixedOutputPath, + ]; } else { const singleAdjustment = audioAdjustments.get(availableAudioInputs[0]) ?? { mode: "none", @@ -248,45 +230,26 @@ export async function muxNativeMacRecordingWithAudio( tempoRatio: 1, durationDeltaMs: 0, }; - if (singleAdjustment.mode !== "none") { - const filterParts: string[] = []; - appendSyncedAudioFilter(filterParts, "[1:a]", "aout", singleAdjustment); - args = [ - "-y", - ...inputs, - "-filter_complex", - filterParts.join(";"), - "-map", - "0:v:0", - "-map", - "[aout]", - "-c:v", - "copy", - "-c:a", - "aac", - "-b:a", - "192k", - "-shortest", - mixedOutputPath, - ]; - } else { - args = [ - "-y", - ...inputs, - "-map", - "0:v:0", - "-map", - "1:a:0", - "-c:v", - "copy", - "-c:a", - "aac", - "-b:a", - "192k", - "-shortest", - mixedOutputPath, - ]; - } + const filterParts: string[] = []; + appendSyncedAudioFilter(filterParts, "[1:a]", "aout", singleAdjustment); + args = [ + "-y", + ...inputs, + "-filter_complex", + filterParts.join(";"), + "-map", + "0:v:0", + "-map", + "[aout]", + "-c:v", + "copy", + "-c:a", + "aac", + "-b:a", + "192k", + "-shortest", + mixedOutputPath, + ]; } console.log("[mux] Running ffmpeg:", ffmpegPath, args.join(" ")); diff --git a/electron/ipc/recording/windows.ts b/electron/ipc/recording/windows.ts index 312f31e1..80af6682 100644 --- a/electron/ipc/recording/windows.ts +++ b/electron/ipc/recording/windows.ts @@ -278,58 +278,38 @@ export async function muxNativeWindowsVideoWithAudio( durationDeltaMs: 0, }; - if (pauseFilter || singleAdjustment.mode !== "none") { - const filterParts: string[] = []; - if (pauseFilter) { - filterParts.push(pauseFilter); - } - const srcLabel = pauseFilter ? "[trimmed_audio]" : "[1:a]"; - appendSyncedAudioFilter(filterParts, srcLabel, "aout", singleAdjustment); - - await execFileAsync( - ffmpegPath, - [ - "-y", - ...inputs, - "-filter_complex", - filterParts.join(";"), - "-map", - "0:v:0", - "-map", - "[aout]", - "-c:v", - "copy", - "-c:a", - "aac", - "-b:a", - "192k", - "-shortest", - mixedOutputPath, - ], - { timeout: 120000, maxBuffer: 10 * 1024 * 1024 }, - ); - } else { - await execFileAsync( - ffmpegPath, - [ - "-y", - ...inputs, - "-map", - "0:v:0", - "-map", - "1:a:0", - "-c:v", - "copy", - "-c:a", - "aac", - "-b:a", - "192k", - "-shortest", - mixedOutputPath, - ], - { timeout: 120000, maxBuffer: 10 * 1024 * 1024 }, - ); + // Always route through the filter graph so that aresample=async=1 is + // applied. This corrects progressive clock drift between video and + // audio tracks that a simple duration comparison cannot detect. + const filterParts: string[] = []; + if (pauseFilter) { + filterParts.push(pauseFilter); } + const srcLabel = pauseFilter ? "[trimmed_audio]" : "[1:a]"; + appendSyncedAudioFilter(filterParts, srcLabel, "aout", singleAdjustment); + + await execFileAsync( + ffmpegPath, + [ + "-y", + ...inputs, + "-filter_complex", + filterParts.join(";"), + "-map", + "0:v:0", + "-map", + "[aout]", + "-c:v", + "copy", + "-c:a", + "aac", + "-b:a", + "192k", + "-shortest", + mixedOutputPath, + ], + { timeout: 120000, maxBuffer: 10 * 1024 * 1024 }, + ); } await moveFileWithOverwrite(mixedOutputPath, videoPath); diff --git a/src/hooks/useScreenRecorder.ts b/src/hooks/useScreenRecorder.ts index 5070bd11..5835f943 100644 --- a/src/hooks/useScreenRecorder.ts +++ b/src/hooks/useScreenRecorder.ts @@ -18,7 +18,7 @@ const HIGH_FRAME_RATE_BOOST = 1.7; const DEFAULT_WIDTH = 1920; const DEFAULT_HEIGHT = 1080; const CODEC_ALIGNMENT = 2; -const RECORDER_TIMESLICE_MS = 1000; +const RECORDER_TIMESLICE_MS = 250; const BITS_PER_MEGABIT = 1_000_000; const MIN_FRAME_RATE = 30; const CHROME_MEDIA_SOURCE = "desktop"; @@ -1090,7 +1090,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { const micAudioTrack = microphoneStream.current?.getAudioTracks()[0]; if (systemAudioTrack && micAudioTrack) { - const context = new AudioContext(); + const context = new AudioContext({ sampleRate: 48000 }); mixingContext.current = context; const systemSource = context.createMediaStreamSource( new MediaStream([systemAudioTrack]), diff --git a/src/lib/exporter/audioEncoder.ts b/src/lib/exporter/audioEncoder.ts index 16d919f3..e47e4c36 100644 --- a/src/lib/exporter/audioEncoder.ts +++ b/src/lib/exporter/audioEncoder.ts @@ -555,7 +555,7 @@ export class AudioProcessor { throw new Error("Export cancelled"); } - audioContext = new AudioContext(); + audioContext = new AudioContext({ sampleRate: 48000 }); const currentDestinationNode = audioContext.createMediaStreamDestination(); destinationNode = currentDestinationNode; @@ -721,7 +721,7 @@ export class AudioProcessor { continue; } - if (Math.abs(audioEl.currentTime - targetTimeSec) > 0.3) { + if (Math.abs(audioEl.currentTime - targetTimeSec) > 0.15) { audioEl.currentTime = targetTimeSec; } @@ -729,6 +729,9 @@ export class AudioProcessor { basePlaybackRate: playbackRate, currentTime: audioEl.currentTime, targetTime: targetTimeSec, + toleranceSeconds: 0.008, + correctionWindowSeconds: 0.5, + maxAdjustment: 0.12, }); if (Math.abs(audioEl.playbackRate - syncedPlaybackRate) > 0.0001) { audioEl.playbackRate = syncedPlaybackRate; @@ -748,7 +751,7 @@ export class AudioProcessor { if (isInRegion) { const audioOffset = (currentTimeMs - region.startMs) / 1000; - if (Math.abs(audioEl.currentTime - audioOffset) > 0.3) { + if (Math.abs(audioEl.currentTime - audioOffset) > 0.15) { audioEl.currentTime = audioOffset; } @@ -756,6 +759,9 @@ export class AudioProcessor { basePlaybackRate: playbackRate, currentTime: audioEl.currentTime, targetTime: audioOffset, + toleranceSeconds: 0.008, + correctionWindowSeconds: 0.5, + maxAdjustment: 0.12, }); if (Math.abs(audioEl.playbackRate - syncedPlaybackRate) > 0.0001) { audioEl.playbackRate = syncedPlaybackRate; diff --git a/src/lib/mediaTiming.test.ts b/src/lib/mediaTiming.test.ts index 1f74a259..62885863 100644 --- a/src/lib/mediaTiming.test.ts +++ b/src/lib/mediaTiming.test.ts @@ -23,10 +23,11 @@ describe("clampMediaTimeToDuration", () => { describe("estimateCompanionAudioStartDelaySeconds", () => { it("returns the positive tail gap when companion audio is shorter", () => { expect(estimateCompanionAudioStartDelaySeconds(10, 9.6)).toBeCloseTo(0.4); + expect(estimateCompanionAudioStartDelaySeconds(10, 9.97)).toBeCloseTo(0.03); }); it("ignores tiny or negative differences", () => { - expect(estimateCompanionAudioStartDelaySeconds(10, 9.97)).toBe(0); + expect(estimateCompanionAudioStartDelaySeconds(10, 9.99)).toBe(0); expect(estimateCompanionAudioStartDelaySeconds(10, 10.5)).toBe(0); }); }); diff --git a/src/lib/mediaTiming.ts b/src/lib/mediaTiming.ts index 11b6a50e..ed3c0ff4 100644 --- a/src/lib/mediaTiming.ts +++ b/src/lib/mediaTiming.ts @@ -19,7 +19,7 @@ export function estimateCompanionAudioStartDelaySeconds( const safeAudioDuration = Math.max(0, audioDuration ?? 0); const estimatedDelaySeconds = safeTimelineDuration - safeAudioDuration; - return estimatedDelaySeconds > 0.05 ? estimatedDelaySeconds : 0; + return estimatedDelaySeconds > 0.025 ? estimatedDelaySeconds : 0; } export function getMediaSyncPlaybackRate({