diff --git a/electron/ipc/handlers.ts b/electron/ipc/handlers.ts index a090bbf5..a1e59a4e 100644 --- a/electron/ipc/handlers.ts +++ b/electron/ipc/handlers.ts @@ -1617,6 +1617,84 @@ async function probeMediaDurationSeconds(filePath: string): Promise { return 0 } +type AudioSyncAdjustment = { + mode: 'none' | 'tempo' | 'delay' + delayMs: number + tempoRatio: number + durationDeltaMs: number +} + +function buildAtempoFilters(tempoRatio: number) { + if (!Number.isFinite(tempoRatio) || tempoRatio <= 0) { + return [] + } + + const filters: string[] = [] + let remaining = tempoRatio + + while (remaining < 0.5) { + filters.push('atempo=0.5') + remaining /= 0.5 + } + + while (remaining > 2) { + filters.push('atempo=2.0') + remaining /= 2.0 + } + + if (Math.abs(remaining - 1) > 0.0005) { + filters.push(`atempo=${remaining.toFixed(6)}`) + } + + return filters +} + +function getAudioSyncAdjustment(videoDuration: number, audioDuration: number): AudioSyncAdjustment { + if ( + !Number.isFinite(videoDuration) || + !Number.isFinite(audioDuration) || + videoDuration <= 0 || + audioDuration <= 0 + ) { + return { mode: 'none', delayMs: 0, tempoRatio: 1, durationDeltaMs: 0 } + } + + const durationDeltaMs = Math.round((videoDuration - audioDuration) * 1000) + const absDeltaMs = Math.abs(durationDeltaMs) + if (absDeltaMs <= 50) { + return { mode: 'none', delayMs: 0, tempoRatio: 1, durationDeltaMs } + } + + const tempoRatio = Math.max(0.5, Math.min(2, audioDuration / videoDuration)) + const relativeDelta = absDeltaMs / Math.max(videoDuration * 1000, 1) + + if (relativeDelta <= 0.03 || absDeltaMs <= 1500 || durationDeltaMs < 0) { + return { mode: 'tempo', delayMs: 0, tempoRatio, durationDeltaMs } + } + + return { mode: 'delay', delayMs: durationDeltaMs, tempoRatio: 1, durationDeltaMs } +} + +function appendSyncedAudioFilter( + filterParts: string[], + inputLabel: string, + outputLabel: string, + adjustment: AudioSyncAdjustment, +) { + const filters: string[] = [] + + if (adjustment.mode === 'delay' && adjustment.delayMs > 0) { + filters.push(`adelay=${adjustment.delayMs}|${adjustment.delayMs}`) + } + + if (adjustment.mode === 'tempo') { + filters.push(...buildAtempoFilters(adjustment.tempoRatio)) + } + + filters.push('aresample=async=1:first_pts=0', 'asetpts=PTS-STARTPTS') + filterParts.push(`${inputLabel}${filters.join(',')}[${outputLabel}]`) +} + function sendWhisperModelDownloadProgress( webContents: Electron.WebContents, payload: { status: 'idle' | 'downloading' | 'downloaded' | 'error'; progress: number; path?: string | null; error?: string }, @@ -2551,27 +2629,35 @@ async function muxNativeWindowsVideoWithAudio(videoPath: string, systemAudioPath if (audioInputs.length === 0) return - // Probe durations to compute audio delay offsets. - // If an audio file is shorter than the video it means the audio device - // started delivering samples late (common with Bluetooth / iPhone mics). + // Match each audio track to the captured video duration. + // Small duration deltas are more often clock drift than a true late start, + // so prefer tempo correction there and reserve leading silence for larger gaps. const videoDuration = await probeMediaDurationSeconds(videoPath) - const audioDelays: Map = new Map() + const audioAdjustments: Map = new Map() if (videoDuration > 0) { for (let i = 0; i < audioFilePaths.length; i++) { const audioDuration = await probeMediaDurationSeconds(audioFilePaths[i]) - const delayMs = audioDuration > 0 ? Math.max(0, Math.round((videoDuration - audioDuration) * 1000)) : 0 - audioDelays.set(audioInputs[i], delayMs) - if (delayMs > 0) { - console.log(`[mux-win] ${audioInputs[i]} audio is ${(delayMs / 1000).toFixed(2)}s shorter than video — adding ${delayMs}ms delay`) + const adjustment = getAudioSyncAdjustment(videoDuration, audioDuration) + audioAdjustments.set(audioInputs[i], adjustment) + if (adjustment.mode === 'tempo') { + console.log( + `[mux-win] ${audioInputs[i]} audio differs from video by ${adjustment.durationDeltaMs}ms — applying tempo ratio ${adjustment.tempoRatio.toFixed(6)}`, + ) + } else if (adjustment.mode === 'delay' && adjustment.delayMs > 0) { + console.log( + `[mux-win] ${audioInputs[i]} audio appears to start late by ${adjustment.delayMs}ms — adding leading silence`, + ) } } } const mixedOutputPath = `${videoPath}.muxed.mp4` const normalizedPauseSegments = normalizePauseSegments(pauseSegments) - const systemDelayMs = audioDelays.get('system') ?? 0 - const micDelayMs = audioDelays.get('mic') ?? 0 + const systemAdjustment = + audioAdjustments.get('system') ?? { mode: 'none', delayMs: 0, tempoRatio: 1, durationDeltaMs: 0 } + const micAdjustment = + audioAdjustments.get('mic') ?? { mode: 'none', delayMs: 0, tempoRatio: 1, durationDeltaMs: 0 } if (audioInputs.length === 2) { // Both system + mic audio: mix them @@ -2589,19 +2675,9 @@ async function muxNativeWindowsVideoWithAudio(videoPath: string, systemAudioPath const systemLabel = systemPauseFilter ? '[system_trimmed]' : '[1:a]' const micLabel = micPauseFilter ? '[mic_trimmed]' : '[2:a]' - // Apply delay to compensate for late audio start - if (micDelayMs > 0) { - filterParts.push(`${micLabel}adelay=${micDelayMs}|${micDelayMs},asetpts=PTS-STARTPTS[m]`) - } else { - filterParts.push(`${micLabel}asetpts=PTS-STARTPTS[m]`) - } - - if (systemDelayMs > 0) { - filterParts.push(`${systemLabel}adelay=${systemDelayMs}|${systemDelayMs},asetpts=PTS-STARTPTS[s]`) - filterParts.push(`[s][m]amix=inputs=2:duration=longest:normalize=0[aout]`) - } else { - filterParts.push(`${systemLabel}[m]amix=inputs=2:duration=longest:normalize=0[aout]`) - } + appendSyncedAudioFilter(filterParts, systemLabel, 's', systemAdjustment) + appendSyncedAudioFilter(filterParts, micLabel, 'm', micAdjustment) + filterParts.push('[s][m]amix=inputs=2:duration=longest:normalize=0[aout]') await execFileAsync( ffmpegPath, @@ -2621,19 +2697,17 @@ async function muxNativeWindowsVideoWithAudio(videoPath: string, systemAudioPath ) } else { // Single audio track - const pauseFilter = buildPausedAudioFilter('1:a', 'aout', normalizedPauseSegments) - const singleDelayMs = audioDelays.get(audioInputs[0]) ?? 0 + const pauseFilter = buildPausedAudioFilter('1:a', 'trimmed_audio', normalizedPauseSegments) + const singleAdjustment = + audioAdjustments.get(audioInputs[0]) ?? { mode: 'none', delayMs: 0, tempoRatio: 1, durationDeltaMs: 0 } - if (pauseFilter || singleDelayMs > 0) { + if (pauseFilter || singleAdjustment.mode !== 'none') { const filterParts: string[] = [] if (pauseFilter) { filterParts.push(pauseFilter) } - const srcLabel = pauseFilter ? '[aout]' : '[1:a]' - if (singleDelayMs > 0) { - filterParts.push(`${srcLabel}adelay=${singleDelayMs}|${singleDelayMs},asetpts=PTS-STARTPTS[delayed]`) - } - const outLabel = singleDelayMs > 0 ? '[delayed]' : '[aout]' + const srcLabel = pauseFilter ? '[trimmed_audio]' : '[1:a]' + appendSyncedAudioFilter(filterParts, srcLabel, 'aout', singleAdjustment) await execFileAsync( ffmpegPath, @@ -2642,7 +2716,7 @@ async function muxNativeWindowsVideoWithAudio(videoPath: string, systemAudioPath ...inputs, '-filter_complex', filterParts.join(';'), '-map', '0:v:0', - '-map', outLabel, + '-map', '[aout]', '-c:v', 'copy', '-c:a', 'aac', '-b:a', '192k', @@ -2880,39 +2954,40 @@ async function muxNativeMacRecordingWithAudio( return } - // Probe durations — if audio is shorter than video it means the audio device - // started late (e.g. iPhone mic over Continuity Camera). Add leading silence. + // Match each audio track to the captured video duration. const videoDuration = await probeMediaDurationSeconds(videoPath) - const audioDelays: Map = new Map() + const audioAdjustments: Map = new Map() if (videoDuration > 0) { for (let i = 0; i < audioFilePaths.length; i++) { const audioDuration = await probeMediaDurationSeconds(audioFilePaths[i]) - const delayMs = audioDuration > 0 ? Math.max(0, Math.round((videoDuration - audioDuration) * 1000)) : 0 - audioDelays.set(availableAudioInputs[i], delayMs) - if (delayMs > 0) { - console.log(`[mux] ${availableAudioInputs[i]} audio is ${(delayMs / 1000).toFixed(2)}s shorter than video — adding ${delayMs}ms delay`) + const adjustment = getAudioSyncAdjustment(videoDuration, audioDuration) + audioAdjustments.set(availableAudioInputs[i], adjustment) + if (adjustment.mode === 'tempo') { + console.log( + `[mux] ${availableAudioInputs[i]} audio differs from video by ${adjustment.durationDeltaMs}ms — applying tempo ratio ${adjustment.tempoRatio.toFixed(6)}`, + ) + } else if (adjustment.mode === 'delay' && adjustment.delayMs > 0) { + console.log( + `[mux] ${availableAudioInputs[i]} audio appears to start late by ${adjustment.delayMs}ms — adding leading silence`, + ) } } } - const systemDelayMs = audioDelays.get('system') ?? 0 - const micDelayMs = audioDelays.get('microphone') ?? 0 - const needsFilter = systemDelayMs > 0 || micDelayMs > 0 + const systemAdjustment = + audioAdjustments.get('system') ?? { mode: 'none', delayMs: 0, tempoRatio: 1, durationDeltaMs: 0 } + const micAdjustment = + audioAdjustments.get('microphone') ?? { mode: 'none', delayMs: 0, tempoRatio: 1, durationDeltaMs: 0 } + const needsFilter = systemAdjustment.mode !== 'none' || micAdjustment.mode !== 'none' let args: string[] if (availableAudioInputs.length === 2) { if (needsFilter) { const filterParts: string[] = [] - if (systemDelayMs > 0) { - filterParts.push(`[1:a]adelay=${systemDelayMs}|${systemDelayMs},asetpts=PTS-STARTPTS[s]`) - } - if (micDelayMs > 0) { - filterParts.push(`[2:a]adelay=${micDelayMs}|${micDelayMs},asetpts=PTS-STARTPTS[m]`) - } - const sLabel = systemDelayMs > 0 ? '[s]' : '[1:a]' - const mLabel = micDelayMs > 0 ? '[m]' : '[2:a]' - filterParts.push(`${sLabel}${mLabel}amix=inputs=2:duration=longest:normalize=0[aout]`) + appendSyncedAudioFilter(filterParts, '[1:a]', 's', systemAdjustment) + appendSyncedAudioFilter(filterParts, '[2:a]', 'm', micAdjustment) + filterParts.push('[s][m]amix=inputs=2:duration=longest:normalize=0[aout]') args = [ '-y', ...inputs, @@ -2940,12 +3015,15 @@ async function muxNativeMacRecordingWithAudio( ] } } else { - if (needsFilter) { - const delayMs = systemDelayMs || micDelayMs + const singleAdjustment = + audioAdjustments.get(availableAudioInputs[0]) ?? { mode: 'none', delayMs: 0, tempoRatio: 1, durationDeltaMs: 0 } + if (singleAdjustment.mode !== 'none') { + const filterParts: string[] = [] + appendSyncedAudioFilter(filterParts, '[1:a]', 'aout', singleAdjustment) args = [ '-y', ...inputs, - '-filter_complex', `[1:a]adelay=${delayMs}|${delayMs},asetpts=PTS-STARTPTS[aout]`, + '-filter_complex', filterParts.join(';'), '-map', '0:v:0', '-map', '[aout]', '-c:v', 'copy', @@ -4030,12 +4108,20 @@ body{background:transparent;overflow:hidden;width:100vw;height:100vh} ipcMain.handle('switch-to-editor', () => { console.log('[switch-to-editor] Opening editor window') recordingFinalizing = false + const sourceSelectorWin = getSourceSelectorWindow() + if (sourceSelectorWin && !sourceSelectorWin.isDestroyed()) { + sourceSelectorWin.close() + } createEditorWindow() }) ipcMain.handle('open-editor-early', () => { console.log('[open-editor-early] Opening editor window while recording finalizes') recordingFinalizing = true + const sourceSelectorWin = getSourceSelectorWindow() + if (sourceSelectorWin && !sourceSelectorWin.isDestroyed()) { + sourceSelectorWin.close() + } createEditorWindow() }) diff --git a/electron/native/ScreenCaptureKitRecorder.swift b/electron/native/ScreenCaptureKitRecorder.swift index d97cb304..1d17a64c 100644 --- a/electron/native/ScreenCaptureKitRecorder.swift +++ b/electron/native/ScreenCaptureKitRecorder.swift @@ -17,6 +17,7 @@ struct CaptureConfig: Codable { } let targetCaptureFPS = 60 +let maxInlineAudioTailExtension = CMTime(seconds: 2.0, preferredTimescale: 600) final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { private let queue = DispatchQueue(label: "openscreen.screencapturekit.video") @@ -33,6 +34,8 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { private var lastSampleBuffer: CMSampleBuffer? private var lastVideoPresentationTime: CMTime = .zero private var lastVideoDuration: CMTime = .zero + private var lastInlineAudioPresentationTime: CMTime = .invalid + private var lastInlineAudioDuration: CMTime = .zero private var isRecording = false private var isPaused = false private var pauseStartedHostTime: CMTime? @@ -360,7 +363,6 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { windowValidationTask = nil trackedWindowId = nil - isRecording = false if let activeStream = stream { do { try await activeStream.stopCapture() @@ -369,6 +371,7 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { } } stream = nil + isRecording = false if let originalBuffer = lastSampleBuffer, let videoInput = videoInput { let additionalTime = lastVideoPresentationTime + frameDuration(for: originalBuffer) @@ -378,7 +381,8 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { } } - let endTime = lastVideoPresentationTime + (lastSampleBuffer.map { frameDuration(for: $0) } ?? .zero) + let videoEndTime = lastVideoPresentationTime + (lastSampleBuffer.map { frameDuration(for: $0) } ?? .zero) + let endTime = resolvedCaptureEndTime(videoEndTime: videoEndTime) assetWriter?.endSession(atSourceTime: endTime) videoInput?.markAsFinished() inlineAudioInput?.markAsFinished() @@ -408,6 +412,8 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { lastSampleBuffer = nil lastVideoPresentationTime = .zero lastVideoDuration = .zero + lastInlineAudioPresentationTime = .invalid + lastInlineAudioDuration = .zero frameCount = 0 isPaused = false pauseStartedHostTime = nil @@ -466,6 +472,34 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { return CMTime(value: 1, timescale: CMTimeScale(targetCaptureFPS)) } + private func latestInlineAudioEndTime() -> CMTime { + guard lastInlineAudioPresentationTime.isValid else { + return .invalid + } + + if lastInlineAudioDuration.isValid && lastInlineAudioDuration > .zero { + return lastInlineAudioPresentationTime + lastInlineAudioDuration + } + + return lastInlineAudioPresentationTime + } + + private func resolvedCaptureEndTime(videoEndTime: CMTime) -> CMTime { + let inlineAudioEndTime = latestInlineAudioEndTime() + guard inlineAudioEndTime.isValid else { + return videoEndTime + } + + if CMTimeCompare(inlineAudioEndTime, videoEndTime) <= 0 { + return videoEndTime + } + + // Prevent a stray inline-audio timestamp from forcing finishWriting + // to finalize an arbitrarily long tail. + let tailExtension = CMTimeSubtract(inlineAudioEndTime, videoEndTime) + return videoEndTime + CMTimeMinimum(tailExtension, maxInlineAudioTailExtension) + } + private func appendAudioSampleBuffer(_ sampleBuffer: CMSampleBuffer, to input: AVAssetWriterInput, firstSampleTime: inout CMTime?, presentationTime: CMTime) { guard input.isReadyForMoreMediaData else { return } @@ -477,7 +511,11 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { // (computed by adjustedPresentationTime), so use it directly. let timing = CMSampleTimingInfo(duration: sampleBuffer.duration, presentationTimeStamp: presentationTime, decodeTimeStamp: sampleBuffer.decodeTimeStamp) if let retimedSampleBuffer = try? CMSampleBuffer(copying: sampleBuffer, withNewTiming: [timing]) { - input.append(retimedSampleBuffer) + let appended = input.append(retimedSampleBuffer) + if appended, input === inlineAudioInput { + lastInlineAudioPresentationTime = presentationTime + lastInlineAudioDuration = sampleBuffer.duration + } } } diff --git a/electron/native/bin/darwin-arm64/openscreen-screencapturekit-helper b/electron/native/bin/darwin-arm64/openscreen-screencapturekit-helper index fd928679..fcc0ca2c 100755 Binary files a/electron/native/bin/darwin-arm64/openscreen-screencapturekit-helper and b/electron/native/bin/darwin-arm64/openscreen-screencapturekit-helper differ diff --git a/electron/native/bin/darwin-x64/openscreen-screencapturekit-helper b/electron/native/bin/darwin-x64/openscreen-screencapturekit-helper index 8827155e..2ae827e4 100755 Binary files a/electron/native/bin/darwin-x64/openscreen-screencapturekit-helper and b/electron/native/bin/darwin-x64/openscreen-screencapturekit-helper differ diff --git a/src/components/video-editor/VideoEditor.tsx b/src/components/video-editor/VideoEditor.tsx index 7abab355..cea4d784 100644 --- a/src/components/video-editor/VideoEditor.tsx +++ b/src/components/video-editor/VideoEditor.tsx @@ -48,7 +48,7 @@ import { VideoExporter, } from "@/lib/exporter"; import { resolveMediaElementSource } from "@/lib/exporter/localMediaSource"; -import { clampMediaTimeToDuration } from "@/lib/mediaTiming"; +import { clampMediaTimeToDuration, getMediaSyncPlaybackRate } from "@/lib/mediaTiming"; import { matchesShortcut } from "@/lib/shortcuts"; import { type AspectRatio, getAspectRatioValue } from "@/utils/aspectRatioUtils"; import { resolveAutoCaptionSourcePath } from "./autoCaptionSource"; @@ -2961,11 +2961,16 @@ export default function VideoEditor() { // Sync audio playback with video currentTime and isPlaying state useEffect(() => { + const currentTimeMs = currentTime * 1000; + const activeSpeedRegion = speedRegions.find( + (region) => currentTimeMs >= region.startMs && currentTimeMs < region.endMs, + ); + const targetPlaybackRate = activeSpeedRegion ? activeSpeedRegion.speed : 1; + for (const region of audioRegions) { const audio = audioElementsRef.current.get(region.id); if (!audio) continue; - const currentTimeMs = currentTime * 1000; const isInRegion = currentTimeMs >= region.startMs && currentTimeMs < region.endMs; if (isPlaying && isInRegion) { @@ -2974,6 +2979,14 @@ export default function VideoEditor() { if (Math.abs(audio.currentTime - audioOffset) > 0.2) { audio.currentTime = audioOffset; } + const syncedPlaybackRate = getMediaSyncPlaybackRate({ + basePlaybackRate: targetPlaybackRate, + currentTime: audio.currentTime, + targetTime: audioOffset, + }); + if (Math.abs(audio.playbackRate - syncedPlaybackRate) > 0.001) { + audio.playbackRate = syncedPlaybackRate; + } if (audio.paused) { audio.play().catch(() => undefined); } @@ -2983,7 +2996,7 @@ export default function VideoEditor() { } } } - }, [isPlaying, currentTime, audioRegions]); + }, [isPlaying, currentTime, audioRegions, speedRegions]); useEffect(() => { if (sourceAudioFallbackPaths.length === 0) { @@ -3001,14 +3014,8 @@ export default function VideoEditor() { const driftThreshold = isPlaying ? 0.35 : 0.01; for (const audio of sourceAudioElementsRef.current.values()) { - const targetTime = clampMediaTimeToDuration( - currentTime, - Number.isFinite(audio.duration) ? audio.duration : null, - ); - - if (Math.abs(audio.playbackRate - targetPlaybackRate) > 0.001) { - audio.playbackRate = targetPlaybackRate; - } + const audioDuration = Number.isFinite(audio.duration) ? audio.duration : null; + const targetTime = clampMediaTimeToDuration(currentTime, audioDuration); if (timelineJumped || Math.abs(audio.currentTime - targetTime) > driftThreshold) { try { @@ -3018,7 +3025,16 @@ export default function VideoEditor() { } } - const atEnd = Number.isFinite(audio.duration) && targetTime >= audio.duration; + const syncedPlaybackRate = getMediaSyncPlaybackRate({ + basePlaybackRate: targetPlaybackRate, + currentTime: audio.currentTime, + targetTime, + }); + if (Math.abs(audio.playbackRate - syncedPlaybackRate) > 0.001) { + audio.playbackRate = syncedPlaybackRate; + } + + const atEnd = audioDuration !== null && targetTime >= audioDuration; if (isPlaying && !atEnd) { audio.play().catch(() => undefined); } else if (!audio.paused) { diff --git a/src/lib/exporter/audioEncoder.ts b/src/lib/exporter/audioEncoder.ts index cca0485d..f76ea975 100644 --- a/src/lib/exporter/audioEncoder.ts +++ b/src/lib/exporter/audioEncoder.ts @@ -1,7 +1,8 @@ import { WebDemuxer } from 'web-demuxer' import type { SpeedRegion, TrimRegion, AudioRegion } from '@/components/video-editor/types' -import type { VideoMuxer } from './muxer' +import { clampMediaTimeToDuration, getMediaSyncPlaybackRate } from '@/lib/mediaTiming' import { resolveMediaElementSource } from './localMediaSource' +import type { VideoMuxer } from './muxer' const AUDIO_BITRATE = 128_000 const DECODE_BACKPRESSURE_LIMIT = 20 @@ -44,19 +45,19 @@ export class AudioProcessor { if (done || !chunk) break if (passthroughTimestampOffsetUs === null) { - passthroughTimestampOffsetUs = Math.min(0, chunk.timestamp) - } + passthroughTimestampOffsetUs = chunk.timestamp + } - const normalizedTimestamp = Math.max( - 0, - chunk.timestamp - passthroughTimestampOffsetUs, - ) - const outputChunk = passthroughTimestampOffsetUs === 0 - ? chunk - : this.cloneEncodedAudioChunkWithTimestamp(chunk, normalizedTimestamp) + const normalizedTimestamp = Math.max( + 0, + chunk.timestamp - passthroughTimestampOffsetUs, + ) + const outputChunk = passthroughTimestampOffsetUs === 0 + ? chunk + : this.cloneEncodedAudioChunkWithTimestamp(chunk, normalizedTimestamp) await muxer.addAudioChunk( - outputChunk, + outputChunk, wroteAudio ? undefined : { @@ -212,16 +213,24 @@ export class AudioProcessor { ? demuxer.read('audio', 0, readEndSec) : demuxer.read('audio') + let sourceTimestampOffsetUs: number | null = null + await this.transcodeAudioStream( audioStream as ReadableStream, audioConfig, muxer, { + observeChunkTimestampUs: (timestampUs) => { + if (sourceTimestampOffsetUs === null) { + sourceTimestampOffsetUs = timestampUs + } + }, shouldSkipChunk: (timestampMs) => this.isInTrimRegion(timestampMs, sortedTrims), transformAudioData: (data) => { const timestampMs = data.timestamp / 1000 const trimOffsetMs = this.computeTrimOffset(timestampMs, sortedTrims) - const adjustedTimestampUs = data.timestamp - trimOffsetMs * 1000 + const adjustedTimestampUs = + data.timestamp - (sourceTimestampOffsetUs ?? 0) - trimOffsetMs * 1000 return this.cloneWithTimestamp(data, Math.max(0, adjustedTimestampUs)) }, }, @@ -233,6 +242,7 @@ export class AudioProcessor { audioConfig: AudioDecoderConfig, muxer: VideoMuxer, options: { + observeChunkTimestampUs?: (timestampUs: number) => void shouldSkipChunk?: (timestampMs: number) => boolean transformAudioData?: (data: AudioData) => AudioData | null } = {}, @@ -344,6 +354,7 @@ export class AudioProcessor { const { done, value: chunk } = await reader.read() if (done || !chunk) break + options.observeChunkTimestampUs?.(chunk.timestamp) const timestampMs = chunk.timestamp / 1000 if (options.shouldSkipChunk?.(timestampMs)) continue @@ -572,19 +583,10 @@ export class AudioProcessor { for (const entry of sourceAudioElements) { const audioEl = entry.media - const targetTimeSec = Math.max( - 0, - Math.min( - currentTimeMs / 1000, - Number.isFinite(audioEl.duration) ? audioEl.duration : currentTimeMs / 1000, - ), - ) + const audioDuration = Number.isFinite(audioEl.duration) ? audioEl.duration : null + const targetTimeSec = clampMediaTimeToDuration(currentTimeMs / 1000, audioDuration) - if (Math.abs(audioEl.playbackRate - playbackRate) > 0.0001) { - audioEl.playbackRate = playbackRate - } - - const atEnd = Number.isFinite(audioEl.duration) && targetTimeSec >= audioEl.duration + const atEnd = audioDuration !== null && targetTimeSec >= audioDuration if (atEnd) { if (!audioEl.paused) { audioEl.pause() @@ -592,11 +594,22 @@ export class AudioProcessor { continue } + if (Math.abs(audioEl.currentTime - targetTimeSec) > 0.3) { + audioEl.currentTime = targetTimeSec + } + + const syncedPlaybackRate = getMediaSyncPlaybackRate({ + basePlaybackRate: playbackRate, + currentTime: audioEl.currentTime, + targetTime: targetTimeSec, + }) + if (Math.abs(audioEl.playbackRate - syncedPlaybackRate) > 0.0001) { + audioEl.playbackRate = syncedPlaybackRate + } + if (audioEl.paused) { audioEl.currentTime = targetTimeSec - audioEl.play().catch(() => {}) - } else if (Math.abs(audioEl.currentTime - targetTimeSec) > 0.3) { - audioEl.currentTime = targetTimeSec + audioEl.play().catch(() => undefined) } } @@ -607,11 +620,22 @@ export class AudioProcessor { if (isInRegion) { const audioOffset = (currentTimeMs - region.startMs) / 1000 + if (Math.abs(audioEl.currentTime - audioOffset) > 0.3) { + audioEl.currentTime = audioOffset + } + + const syncedPlaybackRate = getMediaSyncPlaybackRate({ + basePlaybackRate: playbackRate, + currentTime: audioEl.currentTime, + targetTime: audioOffset, + }) + if (Math.abs(audioEl.playbackRate - syncedPlaybackRate) > 0.0001) { + audioEl.playbackRate = syncedPlaybackRate + } + if (audioEl.paused) { audioEl.currentTime = audioOffset - audioEl.play().catch(() => {}) - } else if (Math.abs(audioEl.currentTime - audioOffset) > 0.3) { - audioEl.currentTime = audioOffset + audioEl.play().catch(() => undefined) } } else { if (!audioEl.paused) { diff --git a/src/lib/exporter/forwardFrameSource.ts b/src/lib/exporter/forwardFrameSource.ts index c203075c..a88c6a5f 100644 --- a/src/lib/exporter/forwardFrameSource.ts +++ b/src/lib/exporter/forwardFrameSource.ts @@ -1,4 +1,5 @@ import { WebDemuxer } from "web-demuxer"; +import { getEffectiveVideoStreamDurationSeconds } from "@/lib/mediaTiming"; const DEFAULT_MAX_DECODE_QUEUE = 12; const DEFAULT_MAX_PENDING_FRAMES = 32; @@ -270,7 +271,16 @@ export class ForwardFrameSource { throw new Error("Frame source not initialized"); } - const clampedTargetTime = Math.max(0, Math.min(targetTimeSec, this.metadata.duration || targetTimeSec)); + const clampedTargetTime = Math.max( + 0, + Math.min( + targetTimeSec, + getEffectiveVideoStreamDurationSeconds({ + duration: this.metadata.duration, + streamDuration: this.metadata.streamDuration, + }) || targetTimeSec, + ), + ); if (clampedTargetTime + 0.001 < this.lastTargetTimeSec) { throw new Error("ForwardFrameSource only supports increasing timestamps"); } diff --git a/src/lib/exporter/streamingDecoder.ts b/src/lib/exporter/streamingDecoder.ts index 83c906f8..471ff026 100644 --- a/src/lib/exporter/streamingDecoder.ts +++ b/src/lib/exporter/streamingDecoder.ts @@ -1,5 +1,6 @@ import { WebDemuxer } from 'web-demuxer'; import type { TrimRegion, SpeedRegion } from '@/components/video-editor/types'; +import { getEffectiveVideoStreamDurationSeconds } from '@/lib/mediaTiming'; const DEFAULT_MAX_DECODE_QUEUE = 12; const DEFAULT_MAX_PENDING_FRAMES = 32; @@ -178,8 +179,12 @@ export class StreamingVideoDecoder { const decoderConfig = await this.demuxer.getDecoderConfig('video'); const codec = this.metadata.codec.toLowerCase(); const shouldPreferSoftwareDecode = codec.includes('av01') || codec.includes('av1'); + const effectiveVideoDuration = getEffectiveVideoStreamDurationSeconds({ + duration: this.metadata.duration, + streamDuration: this.metadata.streamDuration, + }); const segments = this.splitBySpeed( - this.computeSegments(this.metadata.duration, trimRegions), + this.computeSegments(effectiveVideoDuration, trimRegions), speedRegions ); const segmentOutputFrameCounts = segments.map(segment => @@ -512,7 +517,13 @@ export class StreamingVideoDecoder { getEffectiveDuration(trimRegions?: TrimRegion[], speedRegions?: SpeedRegion[]): number { if (!this.metadata) throw new Error('Must call loadMetadata() first'); - const trimSegments = this.computeSegments(this.metadata.duration, trimRegions); + const trimSegments = this.computeSegments( + getEffectiveVideoStreamDurationSeconds({ + duration: this.metadata.duration, + streamDuration: this.metadata.streamDuration, + }), + trimRegions, + ); const speedSegments = this.splitBySpeed(trimSegments, speedRegions); return speedSegments.reduce((sum, seg) => sum + (seg.endSec - seg.startSec) / seg.speed, 0); } diff --git a/src/lib/mediaTiming.test.ts b/src/lib/mediaTiming.test.ts index 762d65d0..f06be380 100644 --- a/src/lib/mediaTiming.test.ts +++ b/src/lib/mediaTiming.test.ts @@ -2,7 +2,9 @@ import { describe, expect, it } from "vitest"; import { clampMediaTimeToDuration, + getEffectiveVideoStreamDurationSeconds, getEffectiveRecordingDurationMs, + getMediaSyncPlaybackRate, } from "./mediaTiming"; describe("clampMediaTimeToDuration", () => { @@ -38,4 +40,73 @@ describe("getEffectiveRecordingDurationMs", () => { }), ).toBe(6_000); }); -}); \ No newline at end of file +}); + +describe("getMediaSyncPlaybackRate", () => { + it("returns the base rate when drift is within tolerance", () => { + expect( + getMediaSyncPlaybackRate({ + basePlaybackRate: 1, + currentTime: 10, + targetTime: 10.01, + }), + ).toBe(1); + }); + + it("nudges playback rate toward the target time", () => { + expect( + getMediaSyncPlaybackRate({ + basePlaybackRate: 1, + currentTime: 10, + targetTime: 10.1, + }), + ).toBeCloseTo(1.05); + + expect( + getMediaSyncPlaybackRate({ + basePlaybackRate: 1, + currentTime: 10.1, + targetTime: 10, + }), + ).toBeCloseTo(0.95); + }); + + it("clamps oversized corrections", () => { + expect( + getMediaSyncPlaybackRate({ + basePlaybackRate: 1, + currentTime: 10, + targetTime: 10.5, + }), + ).toBeCloseTo(1.08); + }); +}); + +describe("getEffectiveVideoStreamDurationSeconds", () => { + it("prefers the video stream duration when present", () => { + expect( + getEffectiveVideoStreamDurationSeconds({ + duration: 12, + streamDuration: 11.2, + }), + ).toBe(11.2); + }); + + it("falls back to the container duration when stream duration is missing", () => { + expect( + getEffectiveVideoStreamDurationSeconds({ + duration: 12, + streamDuration: undefined, + }), + ).toBe(12); + }); + + it("returns zero when neither duration is usable", () => { + expect( + getEffectiveVideoStreamDurationSeconds({ + duration: Number.NaN, + streamDuration: 0, + }), + ).toBe(0); + }); +}); diff --git a/src/lib/mediaTiming.ts b/src/lib/mediaTiming.ts index 869ae43c..29583eaf 100644 --- a/src/lib/mediaTiming.ts +++ b/src/lib/mediaTiming.ts @@ -10,6 +10,63 @@ export function clampMediaTimeToDuration( return Math.max(0, Math.min(safeTargetTime, Math.max(0, duration))); } +export function getMediaSyncPlaybackRate({ + basePlaybackRate, + currentTime, + targetTime, + toleranceSeconds = 0.015, + correctionWindowSeconds = 2, + maxAdjustment = 0.08, +}: { + basePlaybackRate: number; + currentTime: number; + targetTime: number; + toleranceSeconds?: number; + correctionWindowSeconds?: number; + maxAdjustment?: number; +}): number { + const safeBasePlaybackRate = + Number.isFinite(basePlaybackRate) && basePlaybackRate > 0 + ? basePlaybackRate + : 1; + + if (!Number.isFinite(currentTime) || !Number.isFinite(targetTime)) { + return safeBasePlaybackRate; + } + + const driftSeconds = targetTime - currentTime; + if (Math.abs(driftSeconds) <= toleranceSeconds) { + return safeBasePlaybackRate; + } + + const safeCorrectionWindow = correctionWindowSeconds > 0 ? correctionWindowSeconds : 2; + const safeMaxAdjustment = Math.max(0, maxAdjustment); + const adjustment = Math.max( + -safeMaxAdjustment, + Math.min(safeMaxAdjustment, driftSeconds / safeCorrectionWindow), + ); + + return Math.max(0.1, safeBasePlaybackRate + adjustment); +} + +export function getEffectiveVideoStreamDurationSeconds({ + duration, + streamDuration, +}: { + duration?: number | null; + streamDuration?: number | null; +}): number { + if (Number.isFinite(streamDuration) && (streamDuration ?? 0) > 0) { + return Math.max(0, streamDuration ?? 0); + } + + if (Number.isFinite(duration) && (duration ?? 0) > 0) { + return Math.max(0, duration ?? 0); + } + + return 0; +} + export function getEffectiveRecordingDurationMs({ startTimeMs, endTimeMs,