fix(audio): reduce recording and export drift

This commit is contained in:
webadderall
2026-04-04 21:53:25 +11:00
parent 965d4efac7
commit 5fd49fbf0f
10 changed files with 416 additions and 103 deletions
+140 -54
View File
@@ -1617,6 +1617,84 @@ async function probeMediaDurationSeconds(filePath: string): Promise<number> {
return 0
}
type AudioSyncAdjustment = {
mode: 'none' | 'tempo' | 'delay'
delayMs: number
tempoRatio: number
durationDeltaMs: number
}
function buildAtempoFilters(tempoRatio: number) {
if (!Number.isFinite(tempoRatio) || tempoRatio <= 0) {
return []
}
const filters: string[] = []
let remaining = tempoRatio
while (remaining < 0.5) {
filters.push('atempo=0.5')
remaining /= 0.5
}
while (remaining > 2) {
filters.push('atempo=2.0')
remaining /= 2.0
}
if (Math.abs(remaining - 1) > 0.0005) {
filters.push(`atempo=${remaining.toFixed(6)}`)
}
return filters
}
function getAudioSyncAdjustment(videoDuration: number, audioDuration: number): AudioSyncAdjustment {
if (
!Number.isFinite(videoDuration) ||
!Number.isFinite(audioDuration) ||
videoDuration <= 0 ||
audioDuration <= 0
) {
return { mode: 'none', delayMs: 0, tempoRatio: 1, durationDeltaMs: 0 }
}
const durationDeltaMs = Math.round((videoDuration - audioDuration) * 1000)
const absDeltaMs = Math.abs(durationDeltaMs)
if (absDeltaMs <= 50) {
return { mode: 'none', delayMs: 0, tempoRatio: 1, durationDeltaMs }
}
const tempoRatio = Math.max(0.5, Math.min(2, audioDuration / videoDuration))
const relativeDelta = absDeltaMs / Math.max(videoDuration * 1000, 1)
if (relativeDelta <= 0.03 || absDeltaMs <= 1500 || durationDeltaMs < 0) {
return { mode: 'tempo', delayMs: 0, tempoRatio, durationDeltaMs }
}
return { mode: 'delay', delayMs: durationDeltaMs, tempoRatio: 1, durationDeltaMs }
}
function appendSyncedAudioFilter(
filterParts: string[],
inputLabel: string,
outputLabel: string,
adjustment: AudioSyncAdjustment,
) {
const filters: string[] = []
if (adjustment.mode === 'delay' && adjustment.delayMs > 0) {
filters.push(`adelay=${adjustment.delayMs}|${adjustment.delayMs}`)
}
if (adjustment.mode === 'tempo') {
filters.push(...buildAtempoFilters(adjustment.tempoRatio))
}
filters.push('aresample=async=1:first_pts=0', 'asetpts=PTS-STARTPTS')
filterParts.push(`${inputLabel}${filters.join(',')}[${outputLabel}]`)
}
function sendWhisperModelDownloadProgress(
webContents: Electron.WebContents,
payload: { status: 'idle' | 'downloading' | 'downloaded' | 'error'; progress: number; path?: string | null; error?: string },
@@ -2551,27 +2629,35 @@ async function muxNativeWindowsVideoWithAudio(videoPath: string, systemAudioPath
if (audioInputs.length === 0) return
// Probe durations to compute audio delay offsets.
// If an audio file is shorter than the video it means the audio device
// started delivering samples late (common with Bluetooth / iPhone mics).
// Match each audio track to the captured video duration.
// Small duration deltas are more often clock drift than a true late start,
// so prefer tempo correction there and reserve leading silence for larger gaps.
const videoDuration = await probeMediaDurationSeconds(videoPath)
const audioDelays: Map<string, number> = new Map()
const audioAdjustments: Map<string, AudioSyncAdjustment> = new Map()
if (videoDuration > 0) {
for (let i = 0; i < audioFilePaths.length; i++) {
const audioDuration = await probeMediaDurationSeconds(audioFilePaths[i])
const delayMs = audioDuration > 0 ? Math.max(0, Math.round((videoDuration - audioDuration) * 1000)) : 0
audioDelays.set(audioInputs[i], delayMs)
if (delayMs > 0) {
console.log(`[mux-win] ${audioInputs[i]} audio is ${(delayMs / 1000).toFixed(2)}s shorter than video — adding ${delayMs}ms delay`)
const adjustment = getAudioSyncAdjustment(videoDuration, audioDuration)
audioAdjustments.set(audioInputs[i], adjustment)
if (adjustment.mode === 'tempo') {
console.log(
`[mux-win] ${audioInputs[i]} audio differs from video by ${adjustment.durationDeltaMs}ms — applying tempo ratio ${adjustment.tempoRatio.toFixed(6)}`,
)
} else if (adjustment.mode === 'delay' && adjustment.delayMs > 0) {
console.log(
`[mux-win] ${audioInputs[i]} audio appears to start late by ${adjustment.delayMs}ms — adding leading silence`,
)
}
}
}
const mixedOutputPath = `${videoPath}.muxed.mp4`
const normalizedPauseSegments = normalizePauseSegments(pauseSegments)
const systemDelayMs = audioDelays.get('system') ?? 0
const micDelayMs = audioDelays.get('mic') ?? 0
const systemAdjustment =
audioAdjustments.get('system') ?? { mode: 'none', delayMs: 0, tempoRatio: 1, durationDeltaMs: 0 }
const micAdjustment =
audioAdjustments.get('mic') ?? { mode: 'none', delayMs: 0, tempoRatio: 1, durationDeltaMs: 0 }
if (audioInputs.length === 2) {
// Both system + mic audio: mix them
@@ -2589,19 +2675,9 @@ async function muxNativeWindowsVideoWithAudio(videoPath: string, systemAudioPath
const systemLabel = systemPauseFilter ? '[system_trimmed]' : '[1:a]'
const micLabel = micPauseFilter ? '[mic_trimmed]' : '[2:a]'
// Apply delay to compensate for late audio start
if (micDelayMs > 0) {
filterParts.push(`${micLabel}adelay=${micDelayMs}|${micDelayMs},asetpts=PTS-STARTPTS[m]`)
} else {
filterParts.push(`${micLabel}asetpts=PTS-STARTPTS[m]`)
}
if (systemDelayMs > 0) {
filterParts.push(`${systemLabel}adelay=${systemDelayMs}|${systemDelayMs},asetpts=PTS-STARTPTS[s]`)
filterParts.push(`[s][m]amix=inputs=2:duration=longest:normalize=0[aout]`)
} else {
filterParts.push(`${systemLabel}[m]amix=inputs=2:duration=longest:normalize=0[aout]`)
}
appendSyncedAudioFilter(filterParts, systemLabel, 's', systemAdjustment)
appendSyncedAudioFilter(filterParts, micLabel, 'm', micAdjustment)
filterParts.push('[s][m]amix=inputs=2:duration=longest:normalize=0[aout]')
await execFileAsync(
ffmpegPath,
@@ -2621,19 +2697,17 @@ async function muxNativeWindowsVideoWithAudio(videoPath: string, systemAudioPath
)
} else {
// Single audio track
const pauseFilter = buildPausedAudioFilter('1:a', 'aout', normalizedPauseSegments)
const singleDelayMs = audioDelays.get(audioInputs[0]) ?? 0
const pauseFilter = buildPausedAudioFilter('1:a', 'trimmed_audio', normalizedPauseSegments)
const singleAdjustment =
audioAdjustments.get(audioInputs[0]) ?? { mode: 'none', delayMs: 0, tempoRatio: 1, durationDeltaMs: 0 }
if (pauseFilter || singleDelayMs > 0) {
if (pauseFilter || singleAdjustment.mode !== 'none') {
const filterParts: string[] = []
if (pauseFilter) {
filterParts.push(pauseFilter)
}
const srcLabel = pauseFilter ? '[aout]' : '[1:a]'
if (singleDelayMs > 0) {
filterParts.push(`${srcLabel}adelay=${singleDelayMs}|${singleDelayMs},asetpts=PTS-STARTPTS[delayed]`)
}
const outLabel = singleDelayMs > 0 ? '[delayed]' : '[aout]'
const srcLabel = pauseFilter ? '[trimmed_audio]' : '[1:a]'
appendSyncedAudioFilter(filterParts, srcLabel, 'aout', singleAdjustment)
await execFileAsync(
ffmpegPath,
@@ -2642,7 +2716,7 @@ async function muxNativeWindowsVideoWithAudio(videoPath: string, systemAudioPath
...inputs,
'-filter_complex', filterParts.join(';'),
'-map', '0:v:0',
'-map', outLabel,
'-map', '[aout]',
'-c:v', 'copy',
'-c:a', 'aac',
'-b:a', '192k',
@@ -2880,39 +2954,40 @@ async function muxNativeMacRecordingWithAudio(
return
}
// Probe durations — if audio is shorter than video it means the audio device
// started late (e.g. iPhone mic over Continuity Camera). Add leading silence.
// Match each audio track to the captured video duration.
const videoDuration = await probeMediaDurationSeconds(videoPath)
const audioDelays: Map<string, number> = new Map()
const audioAdjustments: Map<string, AudioSyncAdjustment> = new Map()
if (videoDuration > 0) {
for (let i = 0; i < audioFilePaths.length; i++) {
const audioDuration = await probeMediaDurationSeconds(audioFilePaths[i])
const delayMs = audioDuration > 0 ? Math.max(0, Math.round((videoDuration - audioDuration) * 1000)) : 0
audioDelays.set(availableAudioInputs[i], delayMs)
if (delayMs > 0) {
console.log(`[mux] ${availableAudioInputs[i]} audio is ${(delayMs / 1000).toFixed(2)}s shorter than video — adding ${delayMs}ms delay`)
const adjustment = getAudioSyncAdjustment(videoDuration, audioDuration)
audioAdjustments.set(availableAudioInputs[i], adjustment)
if (adjustment.mode === 'tempo') {
console.log(
`[mux] ${availableAudioInputs[i]} audio differs from video by ${adjustment.durationDeltaMs}ms — applying tempo ratio ${adjustment.tempoRatio.toFixed(6)}`,
)
} else if (adjustment.mode === 'delay' && adjustment.delayMs > 0) {
console.log(
`[mux] ${availableAudioInputs[i]} audio appears to start late by ${adjustment.delayMs}ms — adding leading silence`,
)
}
}
}
const systemDelayMs = audioDelays.get('system') ?? 0
const micDelayMs = audioDelays.get('microphone') ?? 0
const needsFilter = systemDelayMs > 0 || micDelayMs > 0
const systemAdjustment =
audioAdjustments.get('system') ?? { mode: 'none', delayMs: 0, tempoRatio: 1, durationDeltaMs: 0 }
const micAdjustment =
audioAdjustments.get('microphone') ?? { mode: 'none', delayMs: 0, tempoRatio: 1, durationDeltaMs: 0 }
const needsFilter = systemAdjustment.mode !== 'none' || micAdjustment.mode !== 'none'
let args: string[]
if (availableAudioInputs.length === 2) {
if (needsFilter) {
const filterParts: string[] = []
if (systemDelayMs > 0) {
filterParts.push(`[1:a]adelay=${systemDelayMs}|${systemDelayMs},asetpts=PTS-STARTPTS[s]`)
}
if (micDelayMs > 0) {
filterParts.push(`[2:a]adelay=${micDelayMs}|${micDelayMs},asetpts=PTS-STARTPTS[m]`)
}
const sLabel = systemDelayMs > 0 ? '[s]' : '[1:a]'
const mLabel = micDelayMs > 0 ? '[m]' : '[2:a]'
filterParts.push(`${sLabel}${mLabel}amix=inputs=2:duration=longest:normalize=0[aout]`)
appendSyncedAudioFilter(filterParts, '[1:a]', 's', systemAdjustment)
appendSyncedAudioFilter(filterParts, '[2:a]', 'm', micAdjustment)
filterParts.push('[s][m]amix=inputs=2:duration=longest:normalize=0[aout]')
args = [
'-y',
...inputs,
@@ -2940,12 +3015,15 @@ async function muxNativeMacRecordingWithAudio(
]
}
} else {
if (needsFilter) {
const delayMs = systemDelayMs || micDelayMs
const singleAdjustment =
audioAdjustments.get(availableAudioInputs[0]) ?? { mode: 'none', delayMs: 0, tempoRatio: 1, durationDeltaMs: 0 }
if (singleAdjustment.mode !== 'none') {
const filterParts: string[] = []
appendSyncedAudioFilter(filterParts, '[1:a]', 'aout', singleAdjustment)
args = [
'-y',
...inputs,
'-filter_complex', `[1:a]adelay=${delayMs}|${delayMs},asetpts=PTS-STARTPTS[aout]`,
'-filter_complex', filterParts.join(';'),
'-map', '0:v:0',
'-map', '[aout]',
'-c:v', 'copy',
@@ -4030,12 +4108,20 @@ body{background:transparent;overflow:hidden;width:100vw;height:100vh}
ipcMain.handle('switch-to-editor', () => {
console.log('[switch-to-editor] Opening editor window')
recordingFinalizing = false
const sourceSelectorWin = getSourceSelectorWindow()
if (sourceSelectorWin && !sourceSelectorWin.isDestroyed()) {
sourceSelectorWin.close()
}
createEditorWindow()
})
ipcMain.handle('open-editor-early', () => {
console.log('[open-editor-early] Opening editor window while recording finalizes')
recordingFinalizing = true
const sourceSelectorWin = getSourceSelectorWindow()
if (sourceSelectorWin && !sourceSelectorWin.isDestroyed()) {
sourceSelectorWin.close()
}
createEditorWindow()
})
+41 -3
View File
@@ -17,6 +17,7 @@ struct CaptureConfig: Codable {
}
let targetCaptureFPS = 60
let maxInlineAudioTailExtension = CMTime(seconds: 2.0, preferredTimescale: 600)
final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate {
private let queue = DispatchQueue(label: "openscreen.screencapturekit.video")
@@ -33,6 +34,8 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate {
private var lastSampleBuffer: CMSampleBuffer?
private var lastVideoPresentationTime: CMTime = .zero
private var lastVideoDuration: CMTime = .zero
private var lastInlineAudioPresentationTime: CMTime = .invalid
private var lastInlineAudioDuration: CMTime = .zero
private var isRecording = false
private var isPaused = false
private var pauseStartedHostTime: CMTime?
@@ -360,7 +363,6 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate {
windowValidationTask = nil
trackedWindowId = nil
isRecording = false
if let activeStream = stream {
do {
try await activeStream.stopCapture()
@@ -369,6 +371,7 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate {
}
}
stream = nil
isRecording = false
if let originalBuffer = lastSampleBuffer, let videoInput = videoInput {
let additionalTime = lastVideoPresentationTime + frameDuration(for: originalBuffer)
@@ -378,7 +381,8 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate {
}
}
let endTime = lastVideoPresentationTime + (lastSampleBuffer.map { frameDuration(for: $0) } ?? .zero)
let videoEndTime = lastVideoPresentationTime + (lastSampleBuffer.map { frameDuration(for: $0) } ?? .zero)
let endTime = resolvedCaptureEndTime(videoEndTime: videoEndTime)
assetWriter?.endSession(atSourceTime: endTime)
videoInput?.markAsFinished()
inlineAudioInput?.markAsFinished()
@@ -408,6 +412,8 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate {
lastSampleBuffer = nil
lastVideoPresentationTime = .zero
lastVideoDuration = .zero
lastInlineAudioPresentationTime = .invalid
lastInlineAudioDuration = .zero
frameCount = 0
isPaused = false
pauseStartedHostTime = nil
@@ -466,6 +472,34 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate {
return CMTime(value: 1, timescale: CMTimeScale(targetCaptureFPS))
}
private func latestInlineAudioEndTime() -> CMTime {
guard lastInlineAudioPresentationTime.isValid else {
return .invalid
}
if lastInlineAudioDuration.isValid && lastInlineAudioDuration > .zero {
return lastInlineAudioPresentationTime + lastInlineAudioDuration
}
return lastInlineAudioPresentationTime
}
private func resolvedCaptureEndTime(videoEndTime: CMTime) -> CMTime {
let inlineAudioEndTime = latestInlineAudioEndTime()
guard inlineAudioEndTime.isValid else {
return videoEndTime
}
if CMTimeCompare(inlineAudioEndTime, videoEndTime) <= 0 {
return videoEndTime
}
// Prevent a stray inline-audio timestamp from forcing finishWriting
// to finalize an arbitrarily long tail.
let tailExtension = CMTimeSubtract(inlineAudioEndTime, videoEndTime)
return videoEndTime + CMTimeMinimum(tailExtension, maxInlineAudioTailExtension)
}
private func appendAudioSampleBuffer(_ sampleBuffer: CMSampleBuffer, to input: AVAssetWriterInput, firstSampleTime: inout CMTime?, presentationTime: CMTime) {
guard input.isReadyForMoreMediaData else { return }
@@ -477,7 +511,11 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate {
// (computed by adjustedPresentationTime), so use it directly.
let timing = CMSampleTimingInfo(duration: sampleBuffer.duration, presentationTimeStamp: presentationTime, decodeTimeStamp: sampleBuffer.decodeTimeStamp)
if let retimedSampleBuffer = try? CMSampleBuffer(copying: sampleBuffer, withNewTiming: [timing]) {
input.append(retimedSampleBuffer)
let appended = input.append(retimedSampleBuffer)
if appended, input === inlineAudioInput {
lastInlineAudioPresentationTime = presentationTime
lastInlineAudioDuration = sampleBuffer.duration
}
}
}
+28 -12
View File
@@ -48,7 +48,7 @@ import {
VideoExporter,
} from "@/lib/exporter";
import { resolveMediaElementSource } from "@/lib/exporter/localMediaSource";
import { clampMediaTimeToDuration } from "@/lib/mediaTiming";
import { clampMediaTimeToDuration, getMediaSyncPlaybackRate } from "@/lib/mediaTiming";
import { matchesShortcut } from "@/lib/shortcuts";
import { type AspectRatio, getAspectRatioValue } from "@/utils/aspectRatioUtils";
import { resolveAutoCaptionSourcePath } from "./autoCaptionSource";
@@ -2961,11 +2961,16 @@ export default function VideoEditor() {
// Sync audio playback with video currentTime and isPlaying state
useEffect(() => {
const currentTimeMs = currentTime * 1000;
const activeSpeedRegion = speedRegions.find(
(region) => currentTimeMs >= region.startMs && currentTimeMs < region.endMs,
);
const targetPlaybackRate = activeSpeedRegion ? activeSpeedRegion.speed : 1;
for (const region of audioRegions) {
const audio = audioElementsRef.current.get(region.id);
if (!audio) continue;
const currentTimeMs = currentTime * 1000;
const isInRegion = currentTimeMs >= region.startMs && currentTimeMs < region.endMs;
if (isPlaying && isInRegion) {
@@ -2974,6 +2979,14 @@ export default function VideoEditor() {
if (Math.abs(audio.currentTime - audioOffset) > 0.2) {
audio.currentTime = audioOffset;
}
const syncedPlaybackRate = getMediaSyncPlaybackRate({
basePlaybackRate: targetPlaybackRate,
currentTime: audio.currentTime,
targetTime: audioOffset,
});
if (Math.abs(audio.playbackRate - syncedPlaybackRate) > 0.001) {
audio.playbackRate = syncedPlaybackRate;
}
if (audio.paused) {
audio.play().catch(() => undefined);
}
@@ -2983,7 +2996,7 @@ export default function VideoEditor() {
}
}
}
}, [isPlaying, currentTime, audioRegions]);
}, [isPlaying, currentTime, audioRegions, speedRegions]);
useEffect(() => {
if (sourceAudioFallbackPaths.length === 0) {
@@ -3001,14 +3014,8 @@ export default function VideoEditor() {
const driftThreshold = isPlaying ? 0.35 : 0.01;
for (const audio of sourceAudioElementsRef.current.values()) {
const targetTime = clampMediaTimeToDuration(
currentTime,
Number.isFinite(audio.duration) ? audio.duration : null,
);
if (Math.abs(audio.playbackRate - targetPlaybackRate) > 0.001) {
audio.playbackRate = targetPlaybackRate;
}
const audioDuration = Number.isFinite(audio.duration) ? audio.duration : null;
const targetTime = clampMediaTimeToDuration(currentTime, audioDuration);
if (timelineJumped || Math.abs(audio.currentTime - targetTime) > driftThreshold) {
try {
@@ -3018,7 +3025,16 @@ export default function VideoEditor() {
}
}
const atEnd = Number.isFinite(audio.duration) && targetTime >= audio.duration;
const syncedPlaybackRate = getMediaSyncPlaybackRate({
basePlaybackRate: targetPlaybackRate,
currentTime: audio.currentTime,
targetTime,
});
if (Math.abs(audio.playbackRate - syncedPlaybackRate) > 0.001) {
audio.playbackRate = syncedPlaybackRate;
}
const atEnd = audioDuration !== null && targetTime >= audioDuration;
if (isPlaying && !atEnd) {
audio.play().catch(() => undefined);
} else if (!audio.paused) {
+54 -30
View File
@@ -1,7 +1,8 @@
import { WebDemuxer } from 'web-demuxer'
import type { SpeedRegion, TrimRegion, AudioRegion } from '@/components/video-editor/types'
import type { VideoMuxer } from './muxer'
import { clampMediaTimeToDuration, getMediaSyncPlaybackRate } from '@/lib/mediaTiming'
import { resolveMediaElementSource } from './localMediaSource'
import type { VideoMuxer } from './muxer'
const AUDIO_BITRATE = 128_000
const DECODE_BACKPRESSURE_LIMIT = 20
@@ -44,19 +45,19 @@ export class AudioProcessor {
if (done || !chunk) break
if (passthroughTimestampOffsetUs === null) {
passthroughTimestampOffsetUs = Math.min(0, chunk.timestamp)
}
passthroughTimestampOffsetUs = chunk.timestamp
}
const normalizedTimestamp = Math.max(
0,
chunk.timestamp - passthroughTimestampOffsetUs,
)
const outputChunk = passthroughTimestampOffsetUs === 0
? chunk
: this.cloneEncodedAudioChunkWithTimestamp(chunk, normalizedTimestamp)
const normalizedTimestamp = Math.max(
0,
chunk.timestamp - passthroughTimestampOffsetUs,
)
const outputChunk = passthroughTimestampOffsetUs === 0
? chunk
: this.cloneEncodedAudioChunkWithTimestamp(chunk, normalizedTimestamp)
await muxer.addAudioChunk(
outputChunk,
outputChunk,
wroteAudio
? undefined
: {
@@ -212,16 +213,24 @@ export class AudioProcessor {
? demuxer.read('audio', 0, readEndSec)
: demuxer.read('audio')
let sourceTimestampOffsetUs: number | null = null
await this.transcodeAudioStream(
audioStream as ReadableStream<EncodedAudioChunk>,
audioConfig,
muxer,
{
observeChunkTimestampUs: (timestampUs) => {
if (sourceTimestampOffsetUs === null) {
sourceTimestampOffsetUs = timestampUs
}
},
shouldSkipChunk: (timestampMs) => this.isInTrimRegion(timestampMs, sortedTrims),
transformAudioData: (data) => {
const timestampMs = data.timestamp / 1000
const trimOffsetMs = this.computeTrimOffset(timestampMs, sortedTrims)
const adjustedTimestampUs = data.timestamp - trimOffsetMs * 1000
const adjustedTimestampUs =
data.timestamp - (sourceTimestampOffsetUs ?? 0) - trimOffsetMs * 1000
return this.cloneWithTimestamp(data, Math.max(0, adjustedTimestampUs))
},
},
@@ -233,6 +242,7 @@ export class AudioProcessor {
audioConfig: AudioDecoderConfig,
muxer: VideoMuxer,
options: {
observeChunkTimestampUs?: (timestampUs: number) => void
shouldSkipChunk?: (timestampMs: number) => boolean
transformAudioData?: (data: AudioData) => AudioData | null
} = {},
@@ -344,6 +354,7 @@ export class AudioProcessor {
const { done, value: chunk } = await reader.read()
if (done || !chunk) break
options.observeChunkTimestampUs?.(chunk.timestamp)
const timestampMs = chunk.timestamp / 1000
if (options.shouldSkipChunk?.(timestampMs)) continue
@@ -572,19 +583,10 @@ export class AudioProcessor {
for (const entry of sourceAudioElements) {
const audioEl = entry.media
const targetTimeSec = Math.max(
0,
Math.min(
currentTimeMs / 1000,
Number.isFinite(audioEl.duration) ? audioEl.duration : currentTimeMs / 1000,
),
)
const audioDuration = Number.isFinite(audioEl.duration) ? audioEl.duration : null
const targetTimeSec = clampMediaTimeToDuration(currentTimeMs / 1000, audioDuration)
if (Math.abs(audioEl.playbackRate - playbackRate) > 0.0001) {
audioEl.playbackRate = playbackRate
}
const atEnd = Number.isFinite(audioEl.duration) && targetTimeSec >= audioEl.duration
const atEnd = audioDuration !== null && targetTimeSec >= audioDuration
if (atEnd) {
if (!audioEl.paused) {
audioEl.pause()
@@ -592,11 +594,22 @@ export class AudioProcessor {
continue
}
if (Math.abs(audioEl.currentTime - targetTimeSec) > 0.3) {
audioEl.currentTime = targetTimeSec
}
const syncedPlaybackRate = getMediaSyncPlaybackRate({
basePlaybackRate: playbackRate,
currentTime: audioEl.currentTime,
targetTime: targetTimeSec,
})
if (Math.abs(audioEl.playbackRate - syncedPlaybackRate) > 0.0001) {
audioEl.playbackRate = syncedPlaybackRate
}
if (audioEl.paused) {
audioEl.currentTime = targetTimeSec
audioEl.play().catch(() => {})
} else if (Math.abs(audioEl.currentTime - targetTimeSec) > 0.3) {
audioEl.currentTime = targetTimeSec
audioEl.play().catch(() => undefined)
}
}
@@ -607,11 +620,22 @@ export class AudioProcessor {
if (isInRegion) {
const audioOffset = (currentTimeMs - region.startMs) / 1000
if (Math.abs(audioEl.currentTime - audioOffset) > 0.3) {
audioEl.currentTime = audioOffset
}
const syncedPlaybackRate = getMediaSyncPlaybackRate({
basePlaybackRate: playbackRate,
currentTime: audioEl.currentTime,
targetTime: audioOffset,
})
if (Math.abs(audioEl.playbackRate - syncedPlaybackRate) > 0.0001) {
audioEl.playbackRate = syncedPlaybackRate
}
if (audioEl.paused) {
audioEl.currentTime = audioOffset
audioEl.play().catch(() => {})
} else if (Math.abs(audioEl.currentTime - audioOffset) > 0.3) {
audioEl.currentTime = audioOffset
audioEl.play().catch(() => undefined)
}
} else {
if (!audioEl.paused) {
+11 -1
View File
@@ -1,4 +1,5 @@
import { WebDemuxer } from "web-demuxer";
import { getEffectiveVideoStreamDurationSeconds } from "@/lib/mediaTiming";
const DEFAULT_MAX_DECODE_QUEUE = 12;
const DEFAULT_MAX_PENDING_FRAMES = 32;
@@ -270,7 +271,16 @@ export class ForwardFrameSource {
throw new Error("Frame source not initialized");
}
const clampedTargetTime = Math.max(0, Math.min(targetTimeSec, this.metadata.duration || targetTimeSec));
const clampedTargetTime = Math.max(
0,
Math.min(
targetTimeSec,
getEffectiveVideoStreamDurationSeconds({
duration: this.metadata.duration,
streamDuration: this.metadata.streamDuration,
}) || targetTimeSec,
),
);
if (clampedTargetTime + 0.001 < this.lastTargetTimeSec) {
throw new Error("ForwardFrameSource only supports increasing timestamps");
}
+13 -2
View File
@@ -1,5 +1,6 @@
import { WebDemuxer } from 'web-demuxer';
import type { TrimRegion, SpeedRegion } from '@/components/video-editor/types';
import { getEffectiveVideoStreamDurationSeconds } from '@/lib/mediaTiming';
const DEFAULT_MAX_DECODE_QUEUE = 12;
const DEFAULT_MAX_PENDING_FRAMES = 32;
@@ -178,8 +179,12 @@ export class StreamingVideoDecoder {
const decoderConfig = await this.demuxer.getDecoderConfig('video');
const codec = this.metadata.codec.toLowerCase();
const shouldPreferSoftwareDecode = codec.includes('av01') || codec.includes('av1');
const effectiveVideoDuration = getEffectiveVideoStreamDurationSeconds({
duration: this.metadata.duration,
streamDuration: this.metadata.streamDuration,
});
const segments = this.splitBySpeed(
this.computeSegments(this.metadata.duration, trimRegions),
this.computeSegments(effectiveVideoDuration, trimRegions),
speedRegions
);
const segmentOutputFrameCounts = segments.map(segment =>
@@ -512,7 +517,13 @@ export class StreamingVideoDecoder {
getEffectiveDuration(trimRegions?: TrimRegion[], speedRegions?: SpeedRegion[]): number {
if (!this.metadata) throw new Error('Must call loadMetadata() first');
const trimSegments = this.computeSegments(this.metadata.duration, trimRegions);
const trimSegments = this.computeSegments(
getEffectiveVideoStreamDurationSeconds({
duration: this.metadata.duration,
streamDuration: this.metadata.streamDuration,
}),
trimRegions,
);
const speedSegments = this.splitBySpeed(trimSegments, speedRegions);
return speedSegments.reduce((sum, seg) => sum + (seg.endSec - seg.startSec) / seg.speed, 0);
}
+72 -1
View File
@@ -2,7 +2,9 @@ import { describe, expect, it } from "vitest";
import {
clampMediaTimeToDuration,
getEffectiveVideoStreamDurationSeconds,
getEffectiveRecordingDurationMs,
getMediaSyncPlaybackRate,
} from "./mediaTiming";
describe("clampMediaTimeToDuration", () => {
@@ -38,4 +40,73 @@ describe("getEffectiveRecordingDurationMs", () => {
}),
).toBe(6_000);
});
});
});
describe("getMediaSyncPlaybackRate", () => {
it("returns the base rate when drift is within tolerance", () => {
expect(
getMediaSyncPlaybackRate({
basePlaybackRate: 1,
currentTime: 10,
targetTime: 10.01,
}),
).toBe(1);
});
it("nudges playback rate toward the target time", () => {
expect(
getMediaSyncPlaybackRate({
basePlaybackRate: 1,
currentTime: 10,
targetTime: 10.1,
}),
).toBeCloseTo(1.05);
expect(
getMediaSyncPlaybackRate({
basePlaybackRate: 1,
currentTime: 10.1,
targetTime: 10,
}),
).toBeCloseTo(0.95);
});
it("clamps oversized corrections", () => {
expect(
getMediaSyncPlaybackRate({
basePlaybackRate: 1,
currentTime: 10,
targetTime: 10.5,
}),
).toBeCloseTo(1.08);
});
});
describe("getEffectiveVideoStreamDurationSeconds", () => {
it("prefers the video stream duration when present", () => {
expect(
getEffectiveVideoStreamDurationSeconds({
duration: 12,
streamDuration: 11.2,
}),
).toBe(11.2);
});
it("falls back to the container duration when stream duration is missing", () => {
expect(
getEffectiveVideoStreamDurationSeconds({
duration: 12,
streamDuration: undefined,
}),
).toBe(12);
});
it("returns zero when neither duration is usable", () => {
expect(
getEffectiveVideoStreamDurationSeconds({
duration: Number.NaN,
streamDuration: 0,
}),
).toBe(0);
});
});
+57
View File
@@ -10,6 +10,63 @@ export function clampMediaTimeToDuration(
return Math.max(0, Math.min(safeTargetTime, Math.max(0, duration)));
}
export function getMediaSyncPlaybackRate({
basePlaybackRate,
currentTime,
targetTime,
toleranceSeconds = 0.015,
correctionWindowSeconds = 2,
maxAdjustment = 0.08,
}: {
basePlaybackRate: number;
currentTime: number;
targetTime: number;
toleranceSeconds?: number;
correctionWindowSeconds?: number;
maxAdjustment?: number;
}): number {
const safeBasePlaybackRate =
Number.isFinite(basePlaybackRate) && basePlaybackRate > 0
? basePlaybackRate
: 1;
if (!Number.isFinite(currentTime) || !Number.isFinite(targetTime)) {
return safeBasePlaybackRate;
}
const driftSeconds = targetTime - currentTime;
if (Math.abs(driftSeconds) <= toleranceSeconds) {
return safeBasePlaybackRate;
}
const safeCorrectionWindow = correctionWindowSeconds > 0 ? correctionWindowSeconds : 2;
const safeMaxAdjustment = Math.max(0, maxAdjustment);
const adjustment = Math.max(
-safeMaxAdjustment,
Math.min(safeMaxAdjustment, driftSeconds / safeCorrectionWindow),
);
return Math.max(0.1, safeBasePlaybackRate + adjustment);
}
export function getEffectiveVideoStreamDurationSeconds({
duration,
streamDuration,
}: {
duration?: number | null;
streamDuration?: number | null;
}): number {
if (Number.isFinite(streamDuration) && (streamDuration ?? 0) > 0) {
return Math.max(0, streamDuration ?? 0);
}
if (Number.isFinite(duration) && (duration ?? 0) > 0) {
return Math.max(0, duration ?? 0);
}
return 0;
}
export function getEffectiveRecordingDurationMs({
startTimeMs,
endTimeMs,