mirror of
https://github.com/webadderallorg/Recordly.git
synced 2026-09-24 23:05:49 +00:00
fix(audio): reduce recording and export drift
This commit is contained in:
+140
-54
@@ -1617,6 +1617,84 @@ async function probeMediaDurationSeconds(filePath: string): Promise<number> {
|
||||
return 0
|
||||
}
|
||||
|
||||
type AudioSyncAdjustment = {
|
||||
mode: 'none' | 'tempo' | 'delay'
|
||||
delayMs: number
|
||||
tempoRatio: number
|
||||
durationDeltaMs: number
|
||||
}
|
||||
|
||||
function buildAtempoFilters(tempoRatio: number) {
|
||||
if (!Number.isFinite(tempoRatio) || tempoRatio <= 0) {
|
||||
return []
|
||||
}
|
||||
|
||||
const filters: string[] = []
|
||||
let remaining = tempoRatio
|
||||
|
||||
while (remaining < 0.5) {
|
||||
filters.push('atempo=0.5')
|
||||
remaining /= 0.5
|
||||
}
|
||||
|
||||
while (remaining > 2) {
|
||||
filters.push('atempo=2.0')
|
||||
remaining /= 2.0
|
||||
}
|
||||
|
||||
if (Math.abs(remaining - 1) > 0.0005) {
|
||||
filters.push(`atempo=${remaining.toFixed(6)}`)
|
||||
}
|
||||
|
||||
return filters
|
||||
}
|
||||
|
||||
function getAudioSyncAdjustment(videoDuration: number, audioDuration: number): AudioSyncAdjustment {
|
||||
if (
|
||||
!Number.isFinite(videoDuration) ||
|
||||
!Number.isFinite(audioDuration) ||
|
||||
videoDuration <= 0 ||
|
||||
audioDuration <= 0
|
||||
) {
|
||||
return { mode: 'none', delayMs: 0, tempoRatio: 1, durationDeltaMs: 0 }
|
||||
}
|
||||
|
||||
const durationDeltaMs = Math.round((videoDuration - audioDuration) * 1000)
|
||||
const absDeltaMs = Math.abs(durationDeltaMs)
|
||||
if (absDeltaMs <= 50) {
|
||||
return { mode: 'none', delayMs: 0, tempoRatio: 1, durationDeltaMs }
|
||||
}
|
||||
|
||||
const tempoRatio = Math.max(0.5, Math.min(2, audioDuration / videoDuration))
|
||||
const relativeDelta = absDeltaMs / Math.max(videoDuration * 1000, 1)
|
||||
|
||||
if (relativeDelta <= 0.03 || absDeltaMs <= 1500 || durationDeltaMs < 0) {
|
||||
return { mode: 'tempo', delayMs: 0, tempoRatio, durationDeltaMs }
|
||||
}
|
||||
|
||||
return { mode: 'delay', delayMs: durationDeltaMs, tempoRatio: 1, durationDeltaMs }
|
||||
}
|
||||
|
||||
function appendSyncedAudioFilter(
|
||||
filterParts: string[],
|
||||
inputLabel: string,
|
||||
outputLabel: string,
|
||||
adjustment: AudioSyncAdjustment,
|
||||
) {
|
||||
const filters: string[] = []
|
||||
|
||||
if (adjustment.mode === 'delay' && adjustment.delayMs > 0) {
|
||||
filters.push(`adelay=${adjustment.delayMs}|${adjustment.delayMs}`)
|
||||
}
|
||||
|
||||
if (adjustment.mode === 'tempo') {
|
||||
filters.push(...buildAtempoFilters(adjustment.tempoRatio))
|
||||
}
|
||||
|
||||
filters.push('aresample=async=1:first_pts=0', 'asetpts=PTS-STARTPTS')
|
||||
filterParts.push(`${inputLabel}${filters.join(',')}[${outputLabel}]`)
|
||||
}
|
||||
|
||||
function sendWhisperModelDownloadProgress(
|
||||
webContents: Electron.WebContents,
|
||||
payload: { status: 'idle' | 'downloading' | 'downloaded' | 'error'; progress: number; path?: string | null; error?: string },
|
||||
@@ -2551,27 +2629,35 @@ async function muxNativeWindowsVideoWithAudio(videoPath: string, systemAudioPath
|
||||
|
||||
if (audioInputs.length === 0) return
|
||||
|
||||
// Probe durations to compute audio delay offsets.
|
||||
// If an audio file is shorter than the video it means the audio device
|
||||
// started delivering samples late (common with Bluetooth / iPhone mics).
|
||||
// Match each audio track to the captured video duration.
|
||||
// Small duration deltas are more often clock drift than a true late start,
|
||||
// so prefer tempo correction there and reserve leading silence for larger gaps.
|
||||
const videoDuration = await probeMediaDurationSeconds(videoPath)
|
||||
const audioDelays: Map<string, number> = new Map()
|
||||
const audioAdjustments: Map<string, AudioSyncAdjustment> = new Map()
|
||||
|
||||
if (videoDuration > 0) {
|
||||
for (let i = 0; i < audioFilePaths.length; i++) {
|
||||
const audioDuration = await probeMediaDurationSeconds(audioFilePaths[i])
|
||||
const delayMs = audioDuration > 0 ? Math.max(0, Math.round((videoDuration - audioDuration) * 1000)) : 0
|
||||
audioDelays.set(audioInputs[i], delayMs)
|
||||
if (delayMs > 0) {
|
||||
console.log(`[mux-win] ${audioInputs[i]} audio is ${(delayMs / 1000).toFixed(2)}s shorter than video — adding ${delayMs}ms delay`)
|
||||
const adjustment = getAudioSyncAdjustment(videoDuration, audioDuration)
|
||||
audioAdjustments.set(audioInputs[i], adjustment)
|
||||
if (adjustment.mode === 'tempo') {
|
||||
console.log(
|
||||
`[mux-win] ${audioInputs[i]} audio differs from video by ${adjustment.durationDeltaMs}ms — applying tempo ratio ${adjustment.tempoRatio.toFixed(6)}`,
|
||||
)
|
||||
} else if (adjustment.mode === 'delay' && adjustment.delayMs > 0) {
|
||||
console.log(
|
||||
`[mux-win] ${audioInputs[i]} audio appears to start late by ${adjustment.delayMs}ms — adding leading silence`,
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const mixedOutputPath = `${videoPath}.muxed.mp4`
|
||||
const normalizedPauseSegments = normalizePauseSegments(pauseSegments)
|
||||
const systemDelayMs = audioDelays.get('system') ?? 0
|
||||
const micDelayMs = audioDelays.get('mic') ?? 0
|
||||
const systemAdjustment =
|
||||
audioAdjustments.get('system') ?? { mode: 'none', delayMs: 0, tempoRatio: 1, durationDeltaMs: 0 }
|
||||
const micAdjustment =
|
||||
audioAdjustments.get('mic') ?? { mode: 'none', delayMs: 0, tempoRatio: 1, durationDeltaMs: 0 }
|
||||
|
||||
if (audioInputs.length === 2) {
|
||||
// Both system + mic audio: mix them
|
||||
@@ -2589,19 +2675,9 @@ async function muxNativeWindowsVideoWithAudio(videoPath: string, systemAudioPath
|
||||
const systemLabel = systemPauseFilter ? '[system_trimmed]' : '[1:a]'
|
||||
const micLabel = micPauseFilter ? '[mic_trimmed]' : '[2:a]'
|
||||
|
||||
// Apply delay to compensate for late audio start
|
||||
if (micDelayMs > 0) {
|
||||
filterParts.push(`${micLabel}adelay=${micDelayMs}|${micDelayMs},asetpts=PTS-STARTPTS[m]`)
|
||||
} else {
|
||||
filterParts.push(`${micLabel}asetpts=PTS-STARTPTS[m]`)
|
||||
}
|
||||
|
||||
if (systemDelayMs > 0) {
|
||||
filterParts.push(`${systemLabel}adelay=${systemDelayMs}|${systemDelayMs},asetpts=PTS-STARTPTS[s]`)
|
||||
filterParts.push(`[s][m]amix=inputs=2:duration=longest:normalize=0[aout]`)
|
||||
} else {
|
||||
filterParts.push(`${systemLabel}[m]amix=inputs=2:duration=longest:normalize=0[aout]`)
|
||||
}
|
||||
appendSyncedAudioFilter(filterParts, systemLabel, 's', systemAdjustment)
|
||||
appendSyncedAudioFilter(filterParts, micLabel, 'm', micAdjustment)
|
||||
filterParts.push('[s][m]amix=inputs=2:duration=longest:normalize=0[aout]')
|
||||
|
||||
await execFileAsync(
|
||||
ffmpegPath,
|
||||
@@ -2621,19 +2697,17 @@ async function muxNativeWindowsVideoWithAudio(videoPath: string, systemAudioPath
|
||||
)
|
||||
} else {
|
||||
// Single audio track
|
||||
const pauseFilter = buildPausedAudioFilter('1:a', 'aout', normalizedPauseSegments)
|
||||
const singleDelayMs = audioDelays.get(audioInputs[0]) ?? 0
|
||||
const pauseFilter = buildPausedAudioFilter('1:a', 'trimmed_audio', normalizedPauseSegments)
|
||||
const singleAdjustment =
|
||||
audioAdjustments.get(audioInputs[0]) ?? { mode: 'none', delayMs: 0, tempoRatio: 1, durationDeltaMs: 0 }
|
||||
|
||||
if (pauseFilter || singleDelayMs > 0) {
|
||||
if (pauseFilter || singleAdjustment.mode !== 'none') {
|
||||
const filterParts: string[] = []
|
||||
if (pauseFilter) {
|
||||
filterParts.push(pauseFilter)
|
||||
}
|
||||
const srcLabel = pauseFilter ? '[aout]' : '[1:a]'
|
||||
if (singleDelayMs > 0) {
|
||||
filterParts.push(`${srcLabel}adelay=${singleDelayMs}|${singleDelayMs},asetpts=PTS-STARTPTS[delayed]`)
|
||||
}
|
||||
const outLabel = singleDelayMs > 0 ? '[delayed]' : '[aout]'
|
||||
const srcLabel = pauseFilter ? '[trimmed_audio]' : '[1:a]'
|
||||
appendSyncedAudioFilter(filterParts, srcLabel, 'aout', singleAdjustment)
|
||||
|
||||
await execFileAsync(
|
||||
ffmpegPath,
|
||||
@@ -2642,7 +2716,7 @@ async function muxNativeWindowsVideoWithAudio(videoPath: string, systemAudioPath
|
||||
...inputs,
|
||||
'-filter_complex', filterParts.join(';'),
|
||||
'-map', '0:v:0',
|
||||
'-map', outLabel,
|
||||
'-map', '[aout]',
|
||||
'-c:v', 'copy',
|
||||
'-c:a', 'aac',
|
||||
'-b:a', '192k',
|
||||
@@ -2880,39 +2954,40 @@ async function muxNativeMacRecordingWithAudio(
|
||||
return
|
||||
}
|
||||
|
||||
// Probe durations — if audio is shorter than video it means the audio device
|
||||
// started late (e.g. iPhone mic over Continuity Camera). Add leading silence.
|
||||
// Match each audio track to the captured video duration.
|
||||
const videoDuration = await probeMediaDurationSeconds(videoPath)
|
||||
const audioDelays: Map<string, number> = new Map()
|
||||
const audioAdjustments: Map<string, AudioSyncAdjustment> = new Map()
|
||||
|
||||
if (videoDuration > 0) {
|
||||
for (let i = 0; i < audioFilePaths.length; i++) {
|
||||
const audioDuration = await probeMediaDurationSeconds(audioFilePaths[i])
|
||||
const delayMs = audioDuration > 0 ? Math.max(0, Math.round((videoDuration - audioDuration) * 1000)) : 0
|
||||
audioDelays.set(availableAudioInputs[i], delayMs)
|
||||
if (delayMs > 0) {
|
||||
console.log(`[mux] ${availableAudioInputs[i]} audio is ${(delayMs / 1000).toFixed(2)}s shorter than video — adding ${delayMs}ms delay`)
|
||||
const adjustment = getAudioSyncAdjustment(videoDuration, audioDuration)
|
||||
audioAdjustments.set(availableAudioInputs[i], adjustment)
|
||||
if (adjustment.mode === 'tempo') {
|
||||
console.log(
|
||||
`[mux] ${availableAudioInputs[i]} audio differs from video by ${adjustment.durationDeltaMs}ms — applying tempo ratio ${adjustment.tempoRatio.toFixed(6)}`,
|
||||
)
|
||||
} else if (adjustment.mode === 'delay' && adjustment.delayMs > 0) {
|
||||
console.log(
|
||||
`[mux] ${availableAudioInputs[i]} audio appears to start late by ${adjustment.delayMs}ms — adding leading silence`,
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const systemDelayMs = audioDelays.get('system') ?? 0
|
||||
const micDelayMs = audioDelays.get('microphone') ?? 0
|
||||
const needsFilter = systemDelayMs > 0 || micDelayMs > 0
|
||||
const systemAdjustment =
|
||||
audioAdjustments.get('system') ?? { mode: 'none', delayMs: 0, tempoRatio: 1, durationDeltaMs: 0 }
|
||||
const micAdjustment =
|
||||
audioAdjustments.get('microphone') ?? { mode: 'none', delayMs: 0, tempoRatio: 1, durationDeltaMs: 0 }
|
||||
const needsFilter = systemAdjustment.mode !== 'none' || micAdjustment.mode !== 'none'
|
||||
|
||||
let args: string[]
|
||||
if (availableAudioInputs.length === 2) {
|
||||
if (needsFilter) {
|
||||
const filterParts: string[] = []
|
||||
if (systemDelayMs > 0) {
|
||||
filterParts.push(`[1:a]adelay=${systemDelayMs}|${systemDelayMs},asetpts=PTS-STARTPTS[s]`)
|
||||
}
|
||||
if (micDelayMs > 0) {
|
||||
filterParts.push(`[2:a]adelay=${micDelayMs}|${micDelayMs},asetpts=PTS-STARTPTS[m]`)
|
||||
}
|
||||
const sLabel = systemDelayMs > 0 ? '[s]' : '[1:a]'
|
||||
const mLabel = micDelayMs > 0 ? '[m]' : '[2:a]'
|
||||
filterParts.push(`${sLabel}${mLabel}amix=inputs=2:duration=longest:normalize=0[aout]`)
|
||||
appendSyncedAudioFilter(filterParts, '[1:a]', 's', systemAdjustment)
|
||||
appendSyncedAudioFilter(filterParts, '[2:a]', 'm', micAdjustment)
|
||||
filterParts.push('[s][m]amix=inputs=2:duration=longest:normalize=0[aout]')
|
||||
args = [
|
||||
'-y',
|
||||
...inputs,
|
||||
@@ -2940,12 +3015,15 @@ async function muxNativeMacRecordingWithAudio(
|
||||
]
|
||||
}
|
||||
} else {
|
||||
if (needsFilter) {
|
||||
const delayMs = systemDelayMs || micDelayMs
|
||||
const singleAdjustment =
|
||||
audioAdjustments.get(availableAudioInputs[0]) ?? { mode: 'none', delayMs: 0, tempoRatio: 1, durationDeltaMs: 0 }
|
||||
if (singleAdjustment.mode !== 'none') {
|
||||
const filterParts: string[] = []
|
||||
appendSyncedAudioFilter(filterParts, '[1:a]', 'aout', singleAdjustment)
|
||||
args = [
|
||||
'-y',
|
||||
...inputs,
|
||||
'-filter_complex', `[1:a]adelay=${delayMs}|${delayMs},asetpts=PTS-STARTPTS[aout]`,
|
||||
'-filter_complex', filterParts.join(';'),
|
||||
'-map', '0:v:0',
|
||||
'-map', '[aout]',
|
||||
'-c:v', 'copy',
|
||||
@@ -4030,12 +4108,20 @@ body{background:transparent;overflow:hidden;width:100vw;height:100vh}
|
||||
ipcMain.handle('switch-to-editor', () => {
|
||||
console.log('[switch-to-editor] Opening editor window')
|
||||
recordingFinalizing = false
|
||||
const sourceSelectorWin = getSourceSelectorWindow()
|
||||
if (sourceSelectorWin && !sourceSelectorWin.isDestroyed()) {
|
||||
sourceSelectorWin.close()
|
||||
}
|
||||
createEditorWindow()
|
||||
})
|
||||
|
||||
ipcMain.handle('open-editor-early', () => {
|
||||
console.log('[open-editor-early] Opening editor window while recording finalizes')
|
||||
recordingFinalizing = true
|
||||
const sourceSelectorWin = getSourceSelectorWindow()
|
||||
if (sourceSelectorWin && !sourceSelectorWin.isDestroyed()) {
|
||||
sourceSelectorWin.close()
|
||||
}
|
||||
createEditorWindow()
|
||||
})
|
||||
|
||||
|
||||
@@ -17,6 +17,7 @@ struct CaptureConfig: Codable {
|
||||
}
|
||||
|
||||
let targetCaptureFPS = 60
|
||||
let maxInlineAudioTailExtension = CMTime(seconds: 2.0, preferredTimescale: 600)
|
||||
|
||||
final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate {
|
||||
private let queue = DispatchQueue(label: "openscreen.screencapturekit.video")
|
||||
@@ -33,6 +34,8 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate {
|
||||
private var lastSampleBuffer: CMSampleBuffer?
|
||||
private var lastVideoPresentationTime: CMTime = .zero
|
||||
private var lastVideoDuration: CMTime = .zero
|
||||
private var lastInlineAudioPresentationTime: CMTime = .invalid
|
||||
private var lastInlineAudioDuration: CMTime = .zero
|
||||
private var isRecording = false
|
||||
private var isPaused = false
|
||||
private var pauseStartedHostTime: CMTime?
|
||||
@@ -360,7 +363,6 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate {
|
||||
windowValidationTask = nil
|
||||
trackedWindowId = nil
|
||||
|
||||
isRecording = false
|
||||
if let activeStream = stream {
|
||||
do {
|
||||
try await activeStream.stopCapture()
|
||||
@@ -369,6 +371,7 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate {
|
||||
}
|
||||
}
|
||||
stream = nil
|
||||
isRecording = false
|
||||
|
||||
if let originalBuffer = lastSampleBuffer, let videoInput = videoInput {
|
||||
let additionalTime = lastVideoPresentationTime + frameDuration(for: originalBuffer)
|
||||
@@ -378,7 +381,8 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate {
|
||||
}
|
||||
}
|
||||
|
||||
let endTime = lastVideoPresentationTime + (lastSampleBuffer.map { frameDuration(for: $0) } ?? .zero)
|
||||
let videoEndTime = lastVideoPresentationTime + (lastSampleBuffer.map { frameDuration(for: $0) } ?? .zero)
|
||||
let endTime = resolvedCaptureEndTime(videoEndTime: videoEndTime)
|
||||
assetWriter?.endSession(atSourceTime: endTime)
|
||||
videoInput?.markAsFinished()
|
||||
inlineAudioInput?.markAsFinished()
|
||||
@@ -408,6 +412,8 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate {
|
||||
lastSampleBuffer = nil
|
||||
lastVideoPresentationTime = .zero
|
||||
lastVideoDuration = .zero
|
||||
lastInlineAudioPresentationTime = .invalid
|
||||
lastInlineAudioDuration = .zero
|
||||
frameCount = 0
|
||||
isPaused = false
|
||||
pauseStartedHostTime = nil
|
||||
@@ -466,6 +472,34 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate {
|
||||
return CMTime(value: 1, timescale: CMTimeScale(targetCaptureFPS))
|
||||
}
|
||||
|
||||
private func latestInlineAudioEndTime() -> CMTime {
|
||||
guard lastInlineAudioPresentationTime.isValid else {
|
||||
return .invalid
|
||||
}
|
||||
|
||||
if lastInlineAudioDuration.isValid && lastInlineAudioDuration > .zero {
|
||||
return lastInlineAudioPresentationTime + lastInlineAudioDuration
|
||||
}
|
||||
|
||||
return lastInlineAudioPresentationTime
|
||||
}
|
||||
|
||||
private func resolvedCaptureEndTime(videoEndTime: CMTime) -> CMTime {
|
||||
let inlineAudioEndTime = latestInlineAudioEndTime()
|
||||
guard inlineAudioEndTime.isValid else {
|
||||
return videoEndTime
|
||||
}
|
||||
|
||||
if CMTimeCompare(inlineAudioEndTime, videoEndTime) <= 0 {
|
||||
return videoEndTime
|
||||
}
|
||||
|
||||
// Prevent a stray inline-audio timestamp from forcing finishWriting
|
||||
// to finalize an arbitrarily long tail.
|
||||
let tailExtension = CMTimeSubtract(inlineAudioEndTime, videoEndTime)
|
||||
return videoEndTime + CMTimeMinimum(tailExtension, maxInlineAudioTailExtension)
|
||||
}
|
||||
|
||||
private func appendAudioSampleBuffer(_ sampleBuffer: CMSampleBuffer, to input: AVAssetWriterInput, firstSampleTime: inout CMTime?, presentationTime: CMTime) {
|
||||
guard input.isReadyForMoreMediaData else { return }
|
||||
|
||||
@@ -477,7 +511,11 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate {
|
||||
// (computed by adjustedPresentationTime), so use it directly.
|
||||
let timing = CMSampleTimingInfo(duration: sampleBuffer.duration, presentationTimeStamp: presentationTime, decodeTimeStamp: sampleBuffer.decodeTimeStamp)
|
||||
if let retimedSampleBuffer = try? CMSampleBuffer(copying: sampleBuffer, withNewTiming: [timing]) {
|
||||
input.append(retimedSampleBuffer)
|
||||
let appended = input.append(retimedSampleBuffer)
|
||||
if appended, input === inlineAudioInput {
|
||||
lastInlineAudioPresentationTime = presentationTime
|
||||
lastInlineAudioDuration = sampleBuffer.duration
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Binary file not shown.
Binary file not shown.
@@ -48,7 +48,7 @@ import {
|
||||
VideoExporter,
|
||||
} from "@/lib/exporter";
|
||||
import { resolveMediaElementSource } from "@/lib/exporter/localMediaSource";
|
||||
import { clampMediaTimeToDuration } from "@/lib/mediaTiming";
|
||||
import { clampMediaTimeToDuration, getMediaSyncPlaybackRate } from "@/lib/mediaTiming";
|
||||
import { matchesShortcut } from "@/lib/shortcuts";
|
||||
import { type AspectRatio, getAspectRatioValue } from "@/utils/aspectRatioUtils";
|
||||
import { resolveAutoCaptionSourcePath } from "./autoCaptionSource";
|
||||
@@ -2961,11 +2961,16 @@ export default function VideoEditor() {
|
||||
|
||||
// Sync audio playback with video currentTime and isPlaying state
|
||||
useEffect(() => {
|
||||
const currentTimeMs = currentTime * 1000;
|
||||
const activeSpeedRegion = speedRegions.find(
|
||||
(region) => currentTimeMs >= region.startMs && currentTimeMs < region.endMs,
|
||||
);
|
||||
const targetPlaybackRate = activeSpeedRegion ? activeSpeedRegion.speed : 1;
|
||||
|
||||
for (const region of audioRegions) {
|
||||
const audio = audioElementsRef.current.get(region.id);
|
||||
if (!audio) continue;
|
||||
|
||||
const currentTimeMs = currentTime * 1000;
|
||||
const isInRegion = currentTimeMs >= region.startMs && currentTimeMs < region.endMs;
|
||||
|
||||
if (isPlaying && isInRegion) {
|
||||
@@ -2974,6 +2979,14 @@ export default function VideoEditor() {
|
||||
if (Math.abs(audio.currentTime - audioOffset) > 0.2) {
|
||||
audio.currentTime = audioOffset;
|
||||
}
|
||||
const syncedPlaybackRate = getMediaSyncPlaybackRate({
|
||||
basePlaybackRate: targetPlaybackRate,
|
||||
currentTime: audio.currentTime,
|
||||
targetTime: audioOffset,
|
||||
});
|
||||
if (Math.abs(audio.playbackRate - syncedPlaybackRate) > 0.001) {
|
||||
audio.playbackRate = syncedPlaybackRate;
|
||||
}
|
||||
if (audio.paused) {
|
||||
audio.play().catch(() => undefined);
|
||||
}
|
||||
@@ -2983,7 +2996,7 @@ export default function VideoEditor() {
|
||||
}
|
||||
}
|
||||
}
|
||||
}, [isPlaying, currentTime, audioRegions]);
|
||||
}, [isPlaying, currentTime, audioRegions, speedRegions]);
|
||||
|
||||
useEffect(() => {
|
||||
if (sourceAudioFallbackPaths.length === 0) {
|
||||
@@ -3001,14 +3014,8 @@ export default function VideoEditor() {
|
||||
const driftThreshold = isPlaying ? 0.35 : 0.01;
|
||||
|
||||
for (const audio of sourceAudioElementsRef.current.values()) {
|
||||
const targetTime = clampMediaTimeToDuration(
|
||||
currentTime,
|
||||
Number.isFinite(audio.duration) ? audio.duration : null,
|
||||
);
|
||||
|
||||
if (Math.abs(audio.playbackRate - targetPlaybackRate) > 0.001) {
|
||||
audio.playbackRate = targetPlaybackRate;
|
||||
}
|
||||
const audioDuration = Number.isFinite(audio.duration) ? audio.duration : null;
|
||||
const targetTime = clampMediaTimeToDuration(currentTime, audioDuration);
|
||||
|
||||
if (timelineJumped || Math.abs(audio.currentTime - targetTime) > driftThreshold) {
|
||||
try {
|
||||
@@ -3018,7 +3025,16 @@ export default function VideoEditor() {
|
||||
}
|
||||
}
|
||||
|
||||
const atEnd = Number.isFinite(audio.duration) && targetTime >= audio.duration;
|
||||
const syncedPlaybackRate = getMediaSyncPlaybackRate({
|
||||
basePlaybackRate: targetPlaybackRate,
|
||||
currentTime: audio.currentTime,
|
||||
targetTime,
|
||||
});
|
||||
if (Math.abs(audio.playbackRate - syncedPlaybackRate) > 0.001) {
|
||||
audio.playbackRate = syncedPlaybackRate;
|
||||
}
|
||||
|
||||
const atEnd = audioDuration !== null && targetTime >= audioDuration;
|
||||
if (isPlaying && !atEnd) {
|
||||
audio.play().catch(() => undefined);
|
||||
} else if (!audio.paused) {
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
import { WebDemuxer } from 'web-demuxer'
|
||||
import type { SpeedRegion, TrimRegion, AudioRegion } from '@/components/video-editor/types'
|
||||
import type { VideoMuxer } from './muxer'
|
||||
import { clampMediaTimeToDuration, getMediaSyncPlaybackRate } from '@/lib/mediaTiming'
|
||||
import { resolveMediaElementSource } from './localMediaSource'
|
||||
import type { VideoMuxer } from './muxer'
|
||||
|
||||
const AUDIO_BITRATE = 128_000
|
||||
const DECODE_BACKPRESSURE_LIMIT = 20
|
||||
@@ -44,19 +45,19 @@ export class AudioProcessor {
|
||||
if (done || !chunk) break
|
||||
|
||||
if (passthroughTimestampOffsetUs === null) {
|
||||
passthroughTimestampOffsetUs = Math.min(0, chunk.timestamp)
|
||||
}
|
||||
passthroughTimestampOffsetUs = chunk.timestamp
|
||||
}
|
||||
|
||||
const normalizedTimestamp = Math.max(
|
||||
0,
|
||||
chunk.timestamp - passthroughTimestampOffsetUs,
|
||||
)
|
||||
const outputChunk = passthroughTimestampOffsetUs === 0
|
||||
? chunk
|
||||
: this.cloneEncodedAudioChunkWithTimestamp(chunk, normalizedTimestamp)
|
||||
const normalizedTimestamp = Math.max(
|
||||
0,
|
||||
chunk.timestamp - passthroughTimestampOffsetUs,
|
||||
)
|
||||
const outputChunk = passthroughTimestampOffsetUs === 0
|
||||
? chunk
|
||||
: this.cloneEncodedAudioChunkWithTimestamp(chunk, normalizedTimestamp)
|
||||
|
||||
await muxer.addAudioChunk(
|
||||
outputChunk,
|
||||
outputChunk,
|
||||
wroteAudio
|
||||
? undefined
|
||||
: {
|
||||
@@ -212,16 +213,24 @@ export class AudioProcessor {
|
||||
? demuxer.read('audio', 0, readEndSec)
|
||||
: demuxer.read('audio')
|
||||
|
||||
let sourceTimestampOffsetUs: number | null = null
|
||||
|
||||
await this.transcodeAudioStream(
|
||||
audioStream as ReadableStream<EncodedAudioChunk>,
|
||||
audioConfig,
|
||||
muxer,
|
||||
{
|
||||
observeChunkTimestampUs: (timestampUs) => {
|
||||
if (sourceTimestampOffsetUs === null) {
|
||||
sourceTimestampOffsetUs = timestampUs
|
||||
}
|
||||
},
|
||||
shouldSkipChunk: (timestampMs) => this.isInTrimRegion(timestampMs, sortedTrims),
|
||||
transformAudioData: (data) => {
|
||||
const timestampMs = data.timestamp / 1000
|
||||
const trimOffsetMs = this.computeTrimOffset(timestampMs, sortedTrims)
|
||||
const adjustedTimestampUs = data.timestamp - trimOffsetMs * 1000
|
||||
const adjustedTimestampUs =
|
||||
data.timestamp - (sourceTimestampOffsetUs ?? 0) - trimOffsetMs * 1000
|
||||
return this.cloneWithTimestamp(data, Math.max(0, adjustedTimestampUs))
|
||||
},
|
||||
},
|
||||
@@ -233,6 +242,7 @@ export class AudioProcessor {
|
||||
audioConfig: AudioDecoderConfig,
|
||||
muxer: VideoMuxer,
|
||||
options: {
|
||||
observeChunkTimestampUs?: (timestampUs: number) => void
|
||||
shouldSkipChunk?: (timestampMs: number) => boolean
|
||||
transformAudioData?: (data: AudioData) => AudioData | null
|
||||
} = {},
|
||||
@@ -344,6 +354,7 @@ export class AudioProcessor {
|
||||
const { done, value: chunk } = await reader.read()
|
||||
if (done || !chunk) break
|
||||
|
||||
options.observeChunkTimestampUs?.(chunk.timestamp)
|
||||
const timestampMs = chunk.timestamp / 1000
|
||||
if (options.shouldSkipChunk?.(timestampMs)) continue
|
||||
|
||||
@@ -572,19 +583,10 @@ export class AudioProcessor {
|
||||
|
||||
for (const entry of sourceAudioElements) {
|
||||
const audioEl = entry.media
|
||||
const targetTimeSec = Math.max(
|
||||
0,
|
||||
Math.min(
|
||||
currentTimeMs / 1000,
|
||||
Number.isFinite(audioEl.duration) ? audioEl.duration : currentTimeMs / 1000,
|
||||
),
|
||||
)
|
||||
const audioDuration = Number.isFinite(audioEl.duration) ? audioEl.duration : null
|
||||
const targetTimeSec = clampMediaTimeToDuration(currentTimeMs / 1000, audioDuration)
|
||||
|
||||
if (Math.abs(audioEl.playbackRate - playbackRate) > 0.0001) {
|
||||
audioEl.playbackRate = playbackRate
|
||||
}
|
||||
|
||||
const atEnd = Number.isFinite(audioEl.duration) && targetTimeSec >= audioEl.duration
|
||||
const atEnd = audioDuration !== null && targetTimeSec >= audioDuration
|
||||
if (atEnd) {
|
||||
if (!audioEl.paused) {
|
||||
audioEl.pause()
|
||||
@@ -592,11 +594,22 @@ export class AudioProcessor {
|
||||
continue
|
||||
}
|
||||
|
||||
if (Math.abs(audioEl.currentTime - targetTimeSec) > 0.3) {
|
||||
audioEl.currentTime = targetTimeSec
|
||||
}
|
||||
|
||||
const syncedPlaybackRate = getMediaSyncPlaybackRate({
|
||||
basePlaybackRate: playbackRate,
|
||||
currentTime: audioEl.currentTime,
|
||||
targetTime: targetTimeSec,
|
||||
})
|
||||
if (Math.abs(audioEl.playbackRate - syncedPlaybackRate) > 0.0001) {
|
||||
audioEl.playbackRate = syncedPlaybackRate
|
||||
}
|
||||
|
||||
if (audioEl.paused) {
|
||||
audioEl.currentTime = targetTimeSec
|
||||
audioEl.play().catch(() => {})
|
||||
} else if (Math.abs(audioEl.currentTime - targetTimeSec) > 0.3) {
|
||||
audioEl.currentTime = targetTimeSec
|
||||
audioEl.play().catch(() => undefined)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -607,11 +620,22 @@ export class AudioProcessor {
|
||||
|
||||
if (isInRegion) {
|
||||
const audioOffset = (currentTimeMs - region.startMs) / 1000
|
||||
if (Math.abs(audioEl.currentTime - audioOffset) > 0.3) {
|
||||
audioEl.currentTime = audioOffset
|
||||
}
|
||||
|
||||
const syncedPlaybackRate = getMediaSyncPlaybackRate({
|
||||
basePlaybackRate: playbackRate,
|
||||
currentTime: audioEl.currentTime,
|
||||
targetTime: audioOffset,
|
||||
})
|
||||
if (Math.abs(audioEl.playbackRate - syncedPlaybackRate) > 0.0001) {
|
||||
audioEl.playbackRate = syncedPlaybackRate
|
||||
}
|
||||
|
||||
if (audioEl.paused) {
|
||||
audioEl.currentTime = audioOffset
|
||||
audioEl.play().catch(() => {})
|
||||
} else if (Math.abs(audioEl.currentTime - audioOffset) > 0.3) {
|
||||
audioEl.currentTime = audioOffset
|
||||
audioEl.play().catch(() => undefined)
|
||||
}
|
||||
} else {
|
||||
if (!audioEl.paused) {
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { WebDemuxer } from "web-demuxer";
|
||||
import { getEffectiveVideoStreamDurationSeconds } from "@/lib/mediaTiming";
|
||||
|
||||
const DEFAULT_MAX_DECODE_QUEUE = 12;
|
||||
const DEFAULT_MAX_PENDING_FRAMES = 32;
|
||||
@@ -270,7 +271,16 @@ export class ForwardFrameSource {
|
||||
throw new Error("Frame source not initialized");
|
||||
}
|
||||
|
||||
const clampedTargetTime = Math.max(0, Math.min(targetTimeSec, this.metadata.duration || targetTimeSec));
|
||||
const clampedTargetTime = Math.max(
|
||||
0,
|
||||
Math.min(
|
||||
targetTimeSec,
|
||||
getEffectiveVideoStreamDurationSeconds({
|
||||
duration: this.metadata.duration,
|
||||
streamDuration: this.metadata.streamDuration,
|
||||
}) || targetTimeSec,
|
||||
),
|
||||
);
|
||||
if (clampedTargetTime + 0.001 < this.lastTargetTimeSec) {
|
||||
throw new Error("ForwardFrameSource only supports increasing timestamps");
|
||||
}
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { WebDemuxer } from 'web-demuxer';
|
||||
import type { TrimRegion, SpeedRegion } from '@/components/video-editor/types';
|
||||
import { getEffectiveVideoStreamDurationSeconds } from '@/lib/mediaTiming';
|
||||
|
||||
const DEFAULT_MAX_DECODE_QUEUE = 12;
|
||||
const DEFAULT_MAX_PENDING_FRAMES = 32;
|
||||
@@ -178,8 +179,12 @@ export class StreamingVideoDecoder {
|
||||
const decoderConfig = await this.demuxer.getDecoderConfig('video');
|
||||
const codec = this.metadata.codec.toLowerCase();
|
||||
const shouldPreferSoftwareDecode = codec.includes('av01') || codec.includes('av1');
|
||||
const effectiveVideoDuration = getEffectiveVideoStreamDurationSeconds({
|
||||
duration: this.metadata.duration,
|
||||
streamDuration: this.metadata.streamDuration,
|
||||
});
|
||||
const segments = this.splitBySpeed(
|
||||
this.computeSegments(this.metadata.duration, trimRegions),
|
||||
this.computeSegments(effectiveVideoDuration, trimRegions),
|
||||
speedRegions
|
||||
);
|
||||
const segmentOutputFrameCounts = segments.map(segment =>
|
||||
@@ -512,7 +517,13 @@ export class StreamingVideoDecoder {
|
||||
|
||||
getEffectiveDuration(trimRegions?: TrimRegion[], speedRegions?: SpeedRegion[]): number {
|
||||
if (!this.metadata) throw new Error('Must call loadMetadata() first');
|
||||
const trimSegments = this.computeSegments(this.metadata.duration, trimRegions);
|
||||
const trimSegments = this.computeSegments(
|
||||
getEffectiveVideoStreamDurationSeconds({
|
||||
duration: this.metadata.duration,
|
||||
streamDuration: this.metadata.streamDuration,
|
||||
}),
|
||||
trimRegions,
|
||||
);
|
||||
const speedSegments = this.splitBySpeed(trimSegments, speedRegions);
|
||||
return speedSegments.reduce((sum, seg) => sum + (seg.endSec - seg.startSec) / seg.speed, 0);
|
||||
}
|
||||
|
||||
@@ -2,7 +2,9 @@ import { describe, expect, it } from "vitest";
|
||||
|
||||
import {
|
||||
clampMediaTimeToDuration,
|
||||
getEffectiveVideoStreamDurationSeconds,
|
||||
getEffectiveRecordingDurationMs,
|
||||
getMediaSyncPlaybackRate,
|
||||
} from "./mediaTiming";
|
||||
|
||||
describe("clampMediaTimeToDuration", () => {
|
||||
@@ -38,4 +40,73 @@ describe("getEffectiveRecordingDurationMs", () => {
|
||||
}),
|
||||
).toBe(6_000);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("getMediaSyncPlaybackRate", () => {
|
||||
it("returns the base rate when drift is within tolerance", () => {
|
||||
expect(
|
||||
getMediaSyncPlaybackRate({
|
||||
basePlaybackRate: 1,
|
||||
currentTime: 10,
|
||||
targetTime: 10.01,
|
||||
}),
|
||||
).toBe(1);
|
||||
});
|
||||
|
||||
it("nudges playback rate toward the target time", () => {
|
||||
expect(
|
||||
getMediaSyncPlaybackRate({
|
||||
basePlaybackRate: 1,
|
||||
currentTime: 10,
|
||||
targetTime: 10.1,
|
||||
}),
|
||||
).toBeCloseTo(1.05);
|
||||
|
||||
expect(
|
||||
getMediaSyncPlaybackRate({
|
||||
basePlaybackRate: 1,
|
||||
currentTime: 10.1,
|
||||
targetTime: 10,
|
||||
}),
|
||||
).toBeCloseTo(0.95);
|
||||
});
|
||||
|
||||
it("clamps oversized corrections", () => {
|
||||
expect(
|
||||
getMediaSyncPlaybackRate({
|
||||
basePlaybackRate: 1,
|
||||
currentTime: 10,
|
||||
targetTime: 10.5,
|
||||
}),
|
||||
).toBeCloseTo(1.08);
|
||||
});
|
||||
});
|
||||
|
||||
describe("getEffectiveVideoStreamDurationSeconds", () => {
|
||||
it("prefers the video stream duration when present", () => {
|
||||
expect(
|
||||
getEffectiveVideoStreamDurationSeconds({
|
||||
duration: 12,
|
||||
streamDuration: 11.2,
|
||||
}),
|
||||
).toBe(11.2);
|
||||
});
|
||||
|
||||
it("falls back to the container duration when stream duration is missing", () => {
|
||||
expect(
|
||||
getEffectiveVideoStreamDurationSeconds({
|
||||
duration: 12,
|
||||
streamDuration: undefined,
|
||||
}),
|
||||
).toBe(12);
|
||||
});
|
||||
|
||||
it("returns zero when neither duration is usable", () => {
|
||||
expect(
|
||||
getEffectiveVideoStreamDurationSeconds({
|
||||
duration: Number.NaN,
|
||||
streamDuration: 0,
|
||||
}),
|
||||
).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -10,6 +10,63 @@ export function clampMediaTimeToDuration(
|
||||
return Math.max(0, Math.min(safeTargetTime, Math.max(0, duration)));
|
||||
}
|
||||
|
||||
export function getMediaSyncPlaybackRate({
|
||||
basePlaybackRate,
|
||||
currentTime,
|
||||
targetTime,
|
||||
toleranceSeconds = 0.015,
|
||||
correctionWindowSeconds = 2,
|
||||
maxAdjustment = 0.08,
|
||||
}: {
|
||||
basePlaybackRate: number;
|
||||
currentTime: number;
|
||||
targetTime: number;
|
||||
toleranceSeconds?: number;
|
||||
correctionWindowSeconds?: number;
|
||||
maxAdjustment?: number;
|
||||
}): number {
|
||||
const safeBasePlaybackRate =
|
||||
Number.isFinite(basePlaybackRate) && basePlaybackRate > 0
|
||||
? basePlaybackRate
|
||||
: 1;
|
||||
|
||||
if (!Number.isFinite(currentTime) || !Number.isFinite(targetTime)) {
|
||||
return safeBasePlaybackRate;
|
||||
}
|
||||
|
||||
const driftSeconds = targetTime - currentTime;
|
||||
if (Math.abs(driftSeconds) <= toleranceSeconds) {
|
||||
return safeBasePlaybackRate;
|
||||
}
|
||||
|
||||
const safeCorrectionWindow = correctionWindowSeconds > 0 ? correctionWindowSeconds : 2;
|
||||
const safeMaxAdjustment = Math.max(0, maxAdjustment);
|
||||
const adjustment = Math.max(
|
||||
-safeMaxAdjustment,
|
||||
Math.min(safeMaxAdjustment, driftSeconds / safeCorrectionWindow),
|
||||
);
|
||||
|
||||
return Math.max(0.1, safeBasePlaybackRate + adjustment);
|
||||
}
|
||||
|
||||
export function getEffectiveVideoStreamDurationSeconds({
|
||||
duration,
|
||||
streamDuration,
|
||||
}: {
|
||||
duration?: number | null;
|
||||
streamDuration?: number | null;
|
||||
}): number {
|
||||
if (Number.isFinite(streamDuration) && (streamDuration ?? 0) > 0) {
|
||||
return Math.max(0, streamDuration ?? 0);
|
||||
}
|
||||
|
||||
if (Number.isFinite(duration) && (duration ?? 0) > 0) {
|
||||
return Math.max(0, duration ?? 0);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
export function getEffectiveRecordingDurationMs({
|
||||
startTimeMs,
|
||||
endTimeMs,
|
||||
|
||||
Reference in New Issue
Block a user