From ac5d54cbba44f653925345047b54727a6ec6f500 Mon Sep 17 00:00:00 2001 From: webadderall <131426131+webadderall@users.noreply.github.com> Date: Sun, 13 Sep 2026 18:01:12 +1000 Subject: [PATCH] Refactor audio and streaming exporters into focused modules --- src/lib/exporter/audioEncoder.ts | 1610 +------------------ src/lib/exporter/audioMediaProcessor.ts | 282 ++++ src/lib/exporter/audioProcessorBase.ts | 154 ++ src/lib/exporter/audioProcessorShared.ts | 144 ++ src/lib/exporter/audioTimelineProcessor.ts | 355 ++++ src/lib/exporter/audioTranscodeProcessor.ts | 275 ++++ src/lib/exporter/offlineAudioProcessor.ts | 444 +++++ src/lib/exporter/streamingDecodePipeline.ts | 448 ++++++ src/lib/exporter/streamingDecoder.ts | 643 +------- src/lib/exporter/streamingDecoderSupport.ts | 128 ++ src/lib/exporter/videoTimelineSegments.ts | 72 + 11 files changed, 2354 insertions(+), 2201 deletions(-) create mode 100644 src/lib/exporter/audioMediaProcessor.ts create mode 100644 src/lib/exporter/audioProcessorBase.ts create mode 100644 src/lib/exporter/audioProcessorShared.ts create mode 100644 src/lib/exporter/audioTimelineProcessor.ts create mode 100644 src/lib/exporter/audioTranscodeProcessor.ts create mode 100644 src/lib/exporter/offlineAudioProcessor.ts create mode 100644 src/lib/exporter/streamingDecodePipeline.ts create mode 100644 src/lib/exporter/streamingDecoderSupport.ts create mode 100644 src/lib/exporter/videoTimelineSegments.ts diff --git a/src/lib/exporter/audioEncoder.ts b/src/lib/exporter/audioEncoder.ts index 6f1ca075..5cda0a88 100644 --- a/src/lib/exporter/audioEncoder.ts +++ b/src/lib/exporter/audioEncoder.ts @@ -1,158 +1,28 @@ -import { WebDemuxer } from "web-demuxer"; -import { SOURCE_AUDIO_NORMALIZE_GAIN } from "@/components/video-editor/audio/audioTypes"; +import type { WebDemuxer } from "web-demuxer"; import type { AudioRegion, ClipRegion, SourceAudioTrackSettings, SpeedRegion, - TrimRegion, } from "@/components/video-editor/types"; -import { buildResolvedAudioPlan, SourceTrackId } from "@/lib/exporter/audioRoutingEngine"; -import { estimateCompanionAudioStartDelaySeconds } from "@/lib/mediaTiming"; -import { resolveMediaElementSource } from "./localMediaSource"; -import type { VideoMuxer } from "./muxer"; import { resolveSourceTrackRoutingPolicy } from "./sourceTrackRoutingPolicy"; +import { AudioTranscodeProcessor } from "./audioTranscodeProcessor"; +import { + hasNonDefaultSourceTrackSettings, + MIN_SPEED_REGION_DELTA_MS, + MP4_AUDIO_CODEC, + type TrimLikeRegion, +} from "./audioProcessorShared"; +import type { VideoMuxer } from "./muxer"; -const AUDIO_BITRATE = 128_000; -const DECODE_BACKPRESSURE_LIMIT = 20; -const ENCODE_BACKPRESSURE_LIMIT = 20; -const MIN_SPEED_REGION_DELTA_MS = 0.0001; -const MP4_AUDIO_CODEC = "mp4a.40.2"; -const OFFLINE_AUDIO_SAMPLE_RATE = 48_000; -const OFFLINE_ENCODE_CHUNK_FRAMES = 1024; -const OFFLINE_CHUNK_DURATION_SEC = 30; -const OFFLINE_MIX_SOFT_LIMITER_THRESHOLD = 0.9; -const OFFLINE_MIX_SOFT_LIMITER_CEILING = 0.985; - -function softLimitSample(sample: number): number { - const magnitude = Math.abs(sample); - if (magnitude <= OFFLINE_MIX_SOFT_LIMITER_THRESHOLD) { - return sample; - } - - const sign = sample < 0 ? -1 : 1; - const kneeRange = 1 - OFFLINE_MIX_SOFT_LIMITER_THRESHOLD; - const limitedMagnitude = - OFFLINE_MIX_SOFT_LIMITER_THRESHOLD + - kneeRange * Math.tanh((magnitude - OFFLINE_MIX_SOFT_LIMITER_THRESHOLD) / kneeRange); - return sign * Math.min(OFFLINE_MIX_SOFT_LIMITER_CEILING, limitedMagnitude); -} - -export function softLimitOfflineMixPeaksInPlace(buffer: AudioBuffer): boolean { - let changed = false; - for (let channel = 0; channel < buffer.numberOfChannels; channel += 1) { - const data = buffer.getChannelData(channel); - for (let index = 0; index < data.length; index += 1) { - const sample = data[index]; - if (!Number.isFinite(sample)) { - data[index] = 0; - changed = true; - continue; - } - - const limited = softLimitSample(sample); - if (limited !== sample) { - data[index] = limited; - changed = true; - } - } - } - return changed; -} - -function resolveSourceTrackGain( - sourceAudioTrackSettings: SourceAudioTrackSettings | undefined, - trackId: "mic" | "system" | "mixed", -) { - const settings = sourceAudioTrackSettings?.[trackId]; - if (!settings) { - return 1; - } - const normalizeGain = settings.normalize ? SOURCE_AUDIO_NORMALIZE_GAIN : 1; - return Math.max(0, Math.min(2, settings.volume * normalizeGain)); -} - -export function getSourceTrackIdFromPath(audioPath: string): SourceTrackId { - const normalized = audioPath.toLowerCase(); - // Check for common patterns like .mic., -mic., mic.mp4, etc. - if ( - normalized.includes(".mic.") || - normalized.includes("-mic.") || - normalized.includes("_mic_") || - normalized.includes("/mic.") || - normalized.includes("\\mic.") || - normalized.endsWith("mic.mp4") || - normalized.endsWith("mic.m4a") || - normalized.endsWith("mic.wav") - ) { - return "mic"; - } - if ( - normalized.includes(".system.") || - normalized.includes("-system.") || - normalized.includes("_system_") || - normalized.includes("/system.") || - normalized.includes("\\system.") || - normalized.endsWith("system.mp4") || - normalized.endsWith("system.m4a") || - normalized.endsWith("system.wav") - ) { - return "system"; - } - return "mixed"; -} - -export function hasNonDefaultSourceTrackSettings( - sourceAudioTrackSettings?: SourceAudioTrackSettings, -) { - if (!sourceAudioTrackSettings) { - return false; - } - return Object.values(sourceAudioTrackSettings).some( - (settings) => - Math.abs((settings?.volume ?? 1) - 1) > 0.0005 || Boolean(settings?.normalize), - ); -} - -interface TimelineSlice { - sourceStartMs: number; - sourceEndMs: number; - speed: number; -} - -interface PreparedOfflineRender { - mainBufferEntry: { buffer: AudioBuffer; gain: number } | null; - companionEntries: Array<{ buffer: AudioBuffer; startDelaySec: number; gain: number }>; - regionEntries: Array<{ buffer: AudioBuffer; region: AudioRegion }>; - mutedSourceOutputRangesSec: Array<{ startSec: number; endSec: number }>; - slices: TimelineSlice[]; - outputDurationMs: number; - numChannels: number; -} - -export async function isAacAudioEncodingSupported( - sampleRate = 48_000, - numberOfChannels = 2, -): Promise { - try { - const support = await AudioEncoder.isConfigSupported({ - codec: MP4_AUDIO_CODEC, - sampleRate, - numberOfChannels, - bitrate: AUDIO_BITRATE, - }); - return support.supported === true; - } catch { - return false; - } -} - -type TrimLikeRegion = TrimRegion | ClipRegion; - -export class AudioProcessor { - private cancelled = false; - private onProgress?: (progress: number) => void; +export { + getSourceTrackIdFromPath, + hasNonDefaultSourceTrackSettings, + isAacAudioEncodingSupported, + softLimitOfflineMixPeaksInPlace, +} from "./audioProcessorShared"; +export class AudioProcessor extends AudioTranscodeProcessor { private isPassthroughAudioCodec(codec: string | undefined): boolean { if (!codec) { return false; @@ -166,7 +36,7 @@ export class AudioProcessor { ); } - private async passthroughAudioStream( + protected async passthroughAudioStream( audioStream: ReadableStream, audioConfig: AudioDecoderConfig, muxer: VideoMuxer, @@ -407,1450 +277,4 @@ export class AudioProcessor { } // Legacy trim-only path used when no speed regions are configured. - private async processTrimOnlyAudio( - demuxer: WebDemuxer, - muxer: VideoMuxer, - sortedTrims: TrimLikeRegion[], - readEndSec?: number, - ): Promise { - let audioConfig: AudioDecoderConfig; - try { - audioConfig = (await demuxer.getDecoderConfig("audio")) as AudioDecoderConfig; - } catch { - console.warn("[AudioProcessor] No audio track found, skipping"); - return; - } - - const codecCheck = await AudioDecoder.isConfigSupported(audioConfig); - if (!codecCheck.supported) { - console.warn("[AudioProcessor] Audio codec not supported:", audioConfig.codec); - return; - } - - const audioStream = - typeof readEndSec === "number" - ? demuxer.read("audio", 0, readEndSec) - : demuxer.read("audio"); - - let sourceTimestampOffsetUs: number | null = null; - - await this.transcodeAudioStream( - audioStream as ReadableStream, - audioConfig, - muxer, - { - observeChunkTimestampUs: (timestampUs) => { - if (sourceTimestampOffsetUs === null) { - sourceTimestampOffsetUs = timestampUs; - } - }, - shouldSkipChunk: (timestampMs) => this.isInTrimRegion(timestampMs, sortedTrims), - transformAudioData: (data) => { - const timestampMs = data.timestamp / 1000; - const trimOffsetMs = this.computeTrimOffset(timestampMs, sortedTrims); - const adjustedTimestampUs = - data.timestamp - (sourceTimestampOffsetUs ?? 0) - trimOffsetMs * 1000; - return this.cloneWithTimestamp(data, Math.max(0, adjustedTimestampUs)); - }, - }, - ); - } - - private async transcodeAudioStream( - audioStream: ReadableStream, - audioConfig: AudioDecoderConfig, - muxer: VideoMuxer, - options: { - observeChunkTimestampUs?: (timestampUs: number) => void; - shouldSkipChunk?: (timestampMs: number) => boolean; - transformAudioData?: (data: AudioData) => AudioData | null; - } = {}, - ): Promise { - const pendingFrames: AudioData[] = []; - let decodeError: Error | null = null; - let encodeError: Error | null = null; - let muxError: Error | null = null; - let pendingMuxing = Promise.resolve(); - const capacityWaiters = new Set<() => void>(); - - const notifyCapacityAvailable = () => { - if (capacityWaiters.size === 0) { - return; - } - - const waiters = [...capacityWaiters]; - capacityWaiters.clear(); - for (const resolve of waiters) { - resolve(); - } - }; - - const waitForCapacity = () => - new Promise((resolve) => { - capacityWaiters.add(resolve); - }); - - const failIfNeeded = () => { - if (decodeError) throw decodeError; - if (encodeError) throw encodeError; - if (muxError) throw muxError; - }; - - const pumpEncodedFrames = () => { - while (!this.cancelled && pendingFrames.length > 0) { - if (encodeError || muxError) { - break; - } - if (encoder.encodeQueueSize >= ENCODE_BACKPRESSURE_LIMIT) { - break; - } - - const frame = pendingFrames.shift(); - if (!frame) { - break; - } - - encoder.encode(frame); - frame.close(); - notifyCapacityAvailable(); - } - }; - - const cleanupPendingFrames = () => { - for (const frame of pendingFrames) { - frame.close(); - } - pendingFrames.length = 0; - }; - - const sampleRate = audioConfig.sampleRate || 48_000; - const channels = audioConfig.numberOfChannels || 2; - const encodeConfig: AudioEncoderConfig = { - codec: MP4_AUDIO_CODEC, - sampleRate, - numberOfChannels: channels, - bitrate: AUDIO_BITRATE, - }; - - const encodeSupport = await AudioEncoder.isConfigSupported(encodeConfig); - if (!encodeSupport.supported) { - console.warn("[AudioProcessor] AAC encoding not supported, skipping audio"); - return; - } - - const encoder = new AudioEncoder({ - output: (chunk: EncodedAudioChunk, meta?: EncodedAudioChunkMetadata) => { - pendingMuxing = pendingMuxing - .then(async () => { - if (this.cancelled) { - return; - } - await muxer.addAudioChunk(chunk, meta); - }) - .catch((error) => { - muxError = error instanceof Error ? error : new Error(String(error)); - notifyCapacityAvailable(); - }); - notifyCapacityAvailable(); - }, - error: (error: DOMException) => { - encodeError = new Error(`[AudioProcessor] Encode error: ${error.message}`); - notifyCapacityAvailable(); - }, - }); - - encoder.configure(encodeConfig); - - const decoder = new AudioDecoder({ - output: (data: AudioData) => { - if (this.cancelled || encodeError || muxError) { - data.close(); - return; - } - - const transformed = options.transformAudioData - ? options.transformAudioData(data) - : data; - - if (transformed !== data) { - data.close(); - } - - if (!transformed) { - return; - } - - pendingFrames.push(transformed); - notifyCapacityAvailable(); - }, - error: (error: DOMException) => { - decodeError = new Error(`[AudioProcessor] Decode error: ${error.message}`); - notifyCapacityAvailable(); - }, - }); - decoder.configure(audioConfig); - - let reader: ReadableStreamDefaultReader | null = null; - - try { - reader = audioStream.getReader(); - while (!this.cancelled) { - failIfNeeded(); - - const { done, value: chunk } = await reader.read(); - if (done || !chunk) break; - - options.observeChunkTimestampUs?.(chunk.timestamp); - const timestampMs = chunk.timestamp / 1000; - if (options.shouldSkipChunk?.(timestampMs)) continue; - - decoder.decode(chunk); - pumpEncodedFrames(); - - while ( - !this.cancelled && - (decoder.decodeQueueSize > DECODE_BACKPRESSURE_LIMIT || - pendingFrames.length > DECODE_BACKPRESSURE_LIMIT || - encoder.encodeQueueSize >= ENCODE_BACKPRESSURE_LIMIT) - ) { - failIfNeeded(); - pumpEncodedFrames(); - await waitForCapacity(); - } - } - - if (decoder.state === "configured") { - await decoder.flush(); - } - - while (!this.cancelled && (pendingFrames.length > 0 || encoder.encodeQueueSize > 0)) { - failIfNeeded(); - pumpEncodedFrames(); - if (pendingFrames.length > 0 || encoder.encodeQueueSize > 0) { - await waitForCapacity(); - } - } - - failIfNeeded(); - - if (encoder.state === "configured") { - await encoder.flush(); - } - - await pendingMuxing; - failIfNeeded(); - } finally { - notifyCapacityAvailable(); - if (reader) { - try { - await reader.cancel(); - } catch { - // reader already closed - } - } - - cleanupPendingFrames(); - - if (decoder.state === "configured") { - decoder.close(); - } - - if (encoder.state === "configured") { - encoder.close(); - } - } - - if (this.cancelled) { - return; - } - } - - // ---------- Offline audio rendering pipeline ---------- - // Replaces the old real-time MediaElement+MediaRecorder approach with - // OfflineAudioContext, which renders as fast as the CPU allows instead of - // waiting for 1× real-time playback. - - private async renderAndMuxOfflineAudio( - videoUrl: string, - trimRegions: TrimLikeRegion[], - speedRegions: SpeedRegion[], - audioRegions: AudioRegion[], - sourceAudioFallbackPaths: string[], - sourceAudioFallbackStartDelayMsByPath: Record | undefined, - sourceAudioTrackSettings: SourceAudioTrackSettings | undefined, - clipRegions: ClipRegion[] | undefined, - muxer: VideoMuxer, - ): Promise { - const prepared = await this.prepareOfflineRender( - videoUrl, - trimRegions, - speedRegions, - audioRegions, - sourceAudioFallbackPaths, - sourceAudioFallbackStartDelayMsByPath, - sourceAudioTrackSettings, - clipRegions, - ); - if (this.cancelled) return; - await this.renderAndEncodeChunked(prepared, muxer); - } - - private async prepareOfflineRender( - videoUrl: string, - trimRegions: TrimLikeRegion[], - speedRegions: SpeedRegion[], - audioRegions: AudioRegion[], - sourceAudioFallbackPaths: string[], - sourceAudioFallbackStartDelayMsByPath?: Record, - sourceAudioTrackSettings?: SourceAudioTrackSettings, - clipRegions?: ClipRegion[], - ): Promise { - if (this.cancelled) throw new Error("Export cancelled"); - this.onProgress?.(0); - - const resolvedPlan = buildResolvedAudioPlan({ - videoResource: videoUrl, - sourceAudioFallbackPaths, - audioRegions, - sourceTrackGainById: { - mic: resolveSourceTrackGain(sourceAudioTrackSettings, "mic"), - system: resolveSourceTrackGain(sourceAudioTrackSettings, "system"), - mixed: resolveSourceTrackGain(sourceAudioTrackSettings, "mixed"), - }, - embeddedGain: Math.max( - 0, - Math.min( - 2, - sourceAudioTrackSettings?.mixed - ? resolveSourceTrackGain(sourceAudioTrackSettings, "mixed") - : sourceAudioTrackSettings?.system - ? resolveSourceTrackGain(sourceAudioTrackSettings, "system") - : 1, - ), - ), - }); - - // Decode embedded source audio separately from companion sidecars. - const mainBuffer = resolvedPlan.includeEmbeddedInExport - ? await this.decodeAudioFromUrl(videoUrl) - : null; - const mainBufferGain = resolveSourceTrackGain(sourceAudioTrackSettings, "mixed"); - const mainBufferEntry = mainBuffer ? { buffer: mainBuffer, gain: mainBufferGain } : null; - if (this.cancelled) throw new Error("Export cancelled"); - - // Decode companion / sidecar audio files - const companionEntries: Array<{ - buffer: AudioBuffer; - startDelaySec: number; - gain: number; - }> = []; - const refDuration = - mainBuffer?.duration ?? - (resolvedPlan.playbackPaths.length > 0 ? await this.getMediaDurationSec(videoUrl) : 0); - for (const audioPath of resolvedPlan.playbackPaths) { - if (this.cancelled) throw new Error("Export cancelled"); - const buffer = await this.decodeAudioFromUrl(audioPath); - if (!buffer) continue; - - companionEntries.push({ - buffer, - gain: resolveSourceTrackGain( - sourceAudioTrackSettings, - getSourceTrackIdFromPath(audioPath), - ), - startDelaySec: estimateCompanionAudioStartDelaySeconds( - refDuration, - buffer.duration, - sourceAudioFallbackStartDelayMsByPath?.[audioPath], - ), - }); - } - if (this.cancelled) throw new Error("Export cancelled"); - - // Decode audio region overlay files - const regionEntries: Array<{ buffer: AudioBuffer; region: AudioRegion }> = []; - for (const region of audioRegions) { - if (this.cancelled) throw new Error("Export cancelled"); - const buffer = await this.decodeAudioFromUrl(region.audioPath); - if (buffer) regionEntries.push({ buffer, region }); - } - - this.onProgress?.(0.2); - - // Determine source duration for timeline calculation - const primaryBuffer = mainBufferEntry?.buffer ?? companionEntries[0]?.buffer ?? null; - if (!primaryBuffer && regionEntries.length === 0) { - throw new Error("No decodable audio sources found"); - } - - let sourceDurationSec: number; - if (mainBufferEntry?.buffer) { - sourceDurationSec = mainBufferEntry.buffer.duration; - } else if (resolvedPlan.playbackPaths.length > 0 || regionEntries.length > 0) { - sourceDurationSec = await this.getMediaDurationSec(videoUrl); - } else { - sourceDurationSec = primaryBuffer?.duration ?? 0; - } - const sourceDurationMs = sourceDurationSec * 1000; - - // Build timeline slices (non-trimmed segments with speed info) - const slices = this.buildTimelineSlices(sourceDurationMs, trimRegions, speedRegions); - - let outputDurationMs = 0; - for (const slice of slices) { - outputDurationMs += (slice.sourceEndMs - slice.sourceStartMs) / slice.speed; - } - - // Extend for audio regions that might exceed the video timeline - for (const { region } of regionEntries) { - const regionEndOutput = this.sourceTimeToOutputTime(region.endMs, slices); - outputDurationMs = Math.max(outputDurationMs, regionEndOutput); - } - - const numChannels = Math.min(primaryBuffer?.numberOfChannels ?? 2, 2); - const mutedSourceOutputRangesSec = (clipRegions ?? []) - .filter( - (clip) => - Boolean(clip.muted) && - Number.isFinite(clip.startMs) && - Number.isFinite(clip.endMs) && - clip.endMs > clip.startMs, - ) - .map((clip) => ({ - startSec: Math.max(0, clip.startMs / 1000), - endSec: Math.max(0, clip.endMs / 1000), - })); - - return { - mainBufferEntry, - companionEntries, - regionEntries, - mutedSourceOutputRangesSec, - slices, - outputDurationMs, - numChannels, - }; - } - - // Render timeline in chunks and encode each chunk to the muxer immediately. - // Memory is bounded to ~OFFLINE_CHUNK_DURATION_SEC of PCM per chunk - // instead of holding the entire output buffer in memory. - private async renderAndEncodeChunked( - prepared: PreparedOfflineRender, - muxer: VideoMuxer, - ): Promise { - const { numChannels } = prepared; - const totalOutputSec = Math.max(prepared.outputDurationMs / 1000, 0.01); - - let encodeError: Error | null = null; - let muxError: Error | null = null; - let pendingMuxing = Promise.resolve(); - let wroteFirstChunk = false; - - const encodeConfig: AudioEncoderConfig = { - codec: MP4_AUDIO_CODEC, - sampleRate: OFFLINE_AUDIO_SAMPLE_RATE, - numberOfChannels: numChannels, - bitrate: AUDIO_BITRATE, - }; - - const supported = await AudioEncoder.isConfigSupported(encodeConfig); - if (!supported.supported) { - console.warn("[AudioProcessor] AAC encoding not supported for offline audio"); - return; - } - - const encoder = new AudioEncoder({ - output: (chunk: EncodedAudioChunk, meta?: EncodedAudioChunkMetadata) => { - pendingMuxing = pendingMuxing - .then(async () => { - if (this.cancelled) return; - await muxer.addAudioChunk(chunk, !wroteFirstChunk ? meta : undefined); - wroteFirstChunk = true; - }) - .catch((error) => { - muxError = error instanceof Error ? error : new Error(String(error)); - }); - }, - error: (error: DOMException) => { - encodeError = new Error(`Audio encode error: ${error.message}`); - }, - }); - encoder.configure(encodeConfig); - - try { - await this.renderChunked( - prepared, - totalOutputSec, - async (rendered, outputOffsetSec) => { - if (encodeError) throw encodeError; - if (muxError) throw muxError; - await this.feedBufferToEncoder(encoder, rendered, outputOffsetSec); - }, - ); - - if (encodeError) throw encodeError; - if (muxError) throw muxError; - - if (encoder.state === "configured") { - await encoder.flush(); - } - - await pendingMuxing; - - if (encodeError) throw encodeError; - if (muxError) throw muxError; - } finally { - if (encoder.state === "configured") { - encoder.close(); - } - } - } - - // Render timeline to a WAV blob for the native/FFmpeg export path. - // Processes in chunks to avoid holding the entire output in memory. - private async renderToWavBlobChunked(prepared: PreparedOfflineRender): Promise { - const totalOutputSec = Math.max(prepared.outputDurationMs / 1000, 0.01); - const totalFrames = Math.ceil(totalOutputSec * OFFLINE_AUDIO_SAMPLE_RATE); - const numChannels = prepared.numChannels; - - const header = this.createWavHeader(OFFLINE_AUDIO_SAMPLE_RATE, numChannels, totalFrames); - const pcmParts: ArrayBuffer[] = [header]; - - await this.renderChunked(prepared, totalOutputSec, async (rendered) => { - pcmParts.push(...this.audioBufferToPcmParts(rendered)); - }); - - return new Blob(pcmParts, { type: "audio/wav" }); - } - - // Shared chunked rendering loop. Processes the timeline in - // OFFLINE_CHUNK_DURATION_SEC segments, calling onChunk for each rendered buffer. - private async renderChunked( - prepared: PreparedOfflineRender, - totalOutputSec: number, - onChunk: ( - rendered: AudioBuffer, - outputOffsetSec: number, - chunkIndex: number, - ) => Promise, - ): Promise { - const { slices, numChannels } = prepared; - let outputOffsetSec = 0; - const chunkCount = Math.ceil(totalOutputSec / OFFLINE_CHUNK_DURATION_SEC); - - for (let i = 0; i < chunkCount && !this.cancelled; i++) { - const chunkSec = Math.min(OFFLINE_CHUNK_DURATION_SEC, totalOutputSec - outputOffsetSec); - const chunkFrames = Math.ceil(chunkSec * OFFLINE_AUDIO_SAMPLE_RATE); - - const offlineCtx = new OfflineAudioContext( - numChannels, - chunkFrames, - OFFLINE_AUDIO_SAMPLE_RATE, - ); - - // Schedule main audio - if (prepared.mainBufferEntry) { - this.scheduleBufferThroughTimeline( - offlineCtx, - prepared.mainBufferEntry.buffer, - slices, - 0, - prepared.mainBufferEntry.gain, - outputOffsetSec, - chunkSec, - prepared.mutedSourceOutputRangesSec, - ); - } - - // Schedule companion/sidecar audio - for (const entry of prepared.companionEntries) { - this.scheduleBufferThroughTimeline( - offlineCtx, - entry.buffer, - slices, - entry.startDelaySec, - entry.gain, - outputOffsetSec, - chunkSec, - prepared.mutedSourceOutputRangesSec, - ); - } - - // Schedule audio region overlays - for (const { buffer, region } of prepared.regionEntries) { - this.scheduleRegionForChunk( - offlineCtx, - buffer, - region, - slices, - outputOffsetSec, - chunkSec, - ); - } - - const rendered = await offlineCtx.startRendering(); - if (this.cancelled) break; - softLimitOfflineMixPeaksInPlace(rendered); - - await onChunk(rendered, outputOffsetSec, i); - - outputOffsetSec += chunkSec; - this.onProgress?.(0.3 + (outputOffsetSec / totalOutputSec) * 0.7); - } - } - - // Schedule an audio region overlay clipped to a specific chunk window. - private scheduleRegionForChunk( - ctx: OfflineAudioContext, - buffer: AudioBuffer, - region: AudioRegion, - slices: TimelineSlice[], - chunkOutputStartSec: number, - chunkDurationSec: number, - ): void { - const outputStartMs = this.sourceTimeToOutputTime(region.startMs, slices); - const outputEndMs = this.sourceTimeToOutputTime(region.endMs, slices); - - let localStartSec = outputStartMs / 1000 - chunkOutputStartSec; - let localEndSec = outputEndMs / 1000 - chunkOutputStartSec; - - // Skip if region doesn't overlap with this chunk - if (localEndSec <= 0 || localStartSec >= chunkDurationSec) return; - - // Clip to chunk bounds - let bufferOffsetSec = 0; - if (localStartSec < 0) { - bufferOffsetSec = -localStartSec; - localStartSec = 0; - } - if (localEndSec > chunkDurationSec) { - localEndSec = chunkDurationSec; - } - - const duration = Math.min(localEndSec - localStartSec, buffer.duration - bufferOffsetSec); - if (duration <= 0.001) return; - - const gainNode = ctx.createGain(); - const normalizeGain = region.normalize ? SOURCE_AUDIO_NORMALIZE_GAIN : 1; - gainNode.gain.value = Math.max(0, Math.min(1, region.volume * normalizeGain)); - gainNode.connect(ctx.destination); - - const source = ctx.createBufferSource(); - source.buffer = buffer; - source.connect(gainNode); - source.start(localStartSec, bufferOffsetSec, duration); - } - - // Feed a rendered AudioBuffer chunk to an AudioEncoder with a timestamp offset. - private async feedBufferToEncoder( - encoder: AudioEncoder, - buffer: AudioBuffer, - timestampOffsetSec: number, - ): Promise { - const sampleRate = buffer.sampleRate; - const numChannels = buffer.numberOfChannels; - const totalFrames = buffer.length; - - for ( - let offset = 0; - offset < totalFrames && !this.cancelled; - offset += OFFLINE_ENCODE_CHUNK_FRAMES - ) { - const frameCount = Math.min(OFFLINE_ENCODE_CHUNK_FRAMES, totalFrames - offset); - - const planarData = new Float32Array(frameCount * numChannels); - for (let ch = 0; ch < numChannels; ch++) { - const channelData = buffer.getChannelData(ch); - planarData.set(channelData.subarray(offset, offset + frameCount), ch * frameCount); - } - - const audioData = new AudioData({ - format: "f32-planar", - sampleRate, - numberOfFrames: frameCount, - numberOfChannels: numChannels, - timestamp: Math.round((offset / sampleRate + timestampOffsetSec) * 1_000_000), - data: planarData, - }); - - encoder.encode(audioData); - audioData.close(); - - while (encoder.encodeQueueSize >= ENCODE_BACKPRESSURE_LIMIT && !this.cancelled) { - await new Promise((r) => setTimeout(r, 1)); - } - } - } - - // Decode audio from a URL using streaming WebCodecs decode with bulk fallback. - // Streaming decode avoids holding the full compressed file in memory alongside - // the decoded AudioBuffer, reducing peak memory for large recordings. - private async decodeAudioFromUrl(url: string): Promise { - try { - const buffer = await this.streamDecodeFromUrl(url); - if (buffer) return buffer; - } catch (error) { - console.warn( - "[AudioProcessor] Streaming decode failed, falling back to bulk decode:", - url, - error, - ); - } - return this.bulkDecodeFromUrl(url, OFFLINE_AUDIO_SAMPLE_RATE); - } - - // Streaming decode via WebDemuxer + AudioDecoder. Decodes audio chunk-by-chunk - // without loading the entire compressed file into a contiguous ArrayBuffer. - private async streamDecodeFromUrl(url: string): Promise { - const source = await resolveMediaElementSource(url); - let demuxer: WebDemuxer | null = null; - - try { - const wasmUrl = new URL("./wasm/web-demuxer.wasm", window.location.href).href; - demuxer = new WebDemuxer({ wasmFilePath: wasmUrl }); - await demuxer.load(source.src); - - let audioConfig: AudioDecoderConfig; - try { - audioConfig = (await demuxer.getDecoderConfig("audio")) as AudioDecoderConfig; - } catch { - return null; // No audio track - } - - const sampleRate = audioConfig.sampleRate || 48_000; - const numChannels = Math.min(audioConfig.numberOfChannels || 2, 2); - - // Accumulate decoded PCM per channel - const channelChunks: Float32Array[][] = Array.from({ length: numChannels }, () => []); - let totalFrames = 0; - let decodeError: Error | null = null; - const decodeCapacityWaiters = new Set<() => void>(); - - const notifyDecodeCapacityAvailable = () => { - if (decodeCapacityWaiters.size === 0) { - return; - } - - const waiters = [...decodeCapacityWaiters]; - decodeCapacityWaiters.clear(); - for (const resolve of waiters) { - resolve(); - } - }; - - const waitForDecodeCapacity = () => - new Promise((resolve) => { - decodeCapacityWaiters.add(resolve); - }); - - const decoder = new AudioDecoder({ - output: (data: AudioData) => { - try { - const frames = data.numberOfFrames; - const dataChannels = Math.min(data.numberOfChannels, numChannels); - const format = data.format; - - if (format?.includes("planar")) { - for (let ch = 0; ch < dataChannels; ch++) { - const size = data.allocationSize({ - planeIndex: ch, - }); - const bytes = new ArrayBuffer(size); - data.copyTo(bytes, { planeIndex: ch }); - channelChunks[ch].push(this.rawToFloat32(bytes, format, frames)); - } - } else if (format) { - // Interleaved format — deinterleave into per-channel arrays. - // Use data.numberOfChannels as stride (not capped dataChannels) - // since the raw buffer contains all source channels. - const srcChannels = data.numberOfChannels; - const size = data.allocationSize({ planeIndex: 0 }); - const bytes = new ArrayBuffer(size); - data.copyTo(bytes, { planeIndex: 0 }); - const interleaved = this.rawToFloat32( - bytes, - format, - frames * srcChannels, - ); - for (let ch = 0; ch < dataChannels; ch++) { - const chData = new Float32Array(frames); - for (let i = 0; i < frames; i++) { - chData[i] = interleaved[i * srcChannels + ch]; - } - channelChunks[ch].push(chData); - } - } - - // Fill missing channels with silence - for (let ch = dataChannels; ch < numChannels; ch++) { - channelChunks[ch].push(new Float32Array(frames)); - } - - totalFrames += frames; - } finally { - data.close(); - notifyDecodeCapacityAvailable(); - } - }, - error: (err: DOMException) => { - decodeError = new Error(`Streaming audio decode error: ${err.message}`); - notifyDecodeCapacityAvailable(); - }, - }); - - decoder.configure(audioConfig); - - const audioStream = demuxer.read("audio"); - const reader = (audioStream as ReadableStream).getReader(); - - try { - while (!this.cancelled) { - if (decodeError) throw decodeError; - const { done, value: chunk } = await reader.read(); - if (done || !chunk) break; - - decoder.decode(chunk); - - while (decoder.decodeQueueSize > DECODE_BACKPRESSURE_LIMIT && !this.cancelled) { - if (decodeError) throw decodeError; - await waitForDecodeCapacity(); - } - } - - if (decoder.state === "configured") { - await decoder.flush(); - } - if (decodeError) throw decodeError; - } finally { - notifyDecodeCapacityAvailable(); - try { - await reader.cancel(); - } catch { - /* reader already closed */ - } - if (decoder.state === "configured") { - decoder.close(); - } - } - - if (totalFrames === 0) return null; - - // Build AudioBuffer from accumulated chunks - const audioBuffer = new AudioBuffer({ - length: totalFrames, - numberOfChannels: numChannels, - sampleRate, - }); - for (let ch = 0; ch < numChannels; ch++) { - const channelData = audioBuffer.getChannelData(ch); - let writeOffset = 0; - for (const chunk of channelChunks[ch]) { - channelData.set(chunk, writeOffset); - writeOffset += chunk.length; - } - } - - return audioBuffer; - } finally { - source.revoke(); - try { - demuxer?.destroy(); - } catch { - /* cleanup */ - } - } - } - - // Convert raw bytes from AudioData to Float32Array based on the sample format. - private rawToFloat32(bytes: ArrayBuffer, format: string, sampleCount: number): Float32Array { - if (format.startsWith("f32")) { - return new Float32Array(bytes); - } - if (format.startsWith("s16")) { - const int16 = new Int16Array(bytes); - const f32 = new Float32Array(sampleCount); - for (let i = 0; i < sampleCount; i++) { - f32[i] = int16[i] / 0x8000; - } - return f32; - } - if (format.startsWith("s32")) { - const int32 = new Int32Array(bytes); - const f32 = new Float32Array(sampleCount); - for (let i = 0; i < sampleCount; i++) { - f32[i] = int32[i] / 0x80000000; - } - return f32; - } - if (format.startsWith("u8")) { - const uint8 = new Uint8Array(bytes); - const f32 = new Float32Array(sampleCount); - for (let i = 0; i < sampleCount; i++) { - f32[i] = (uint8[i] - 128) / 128; - } - return f32; - } - // Unknown format — attempt float32 interpretation - return new Float32Array(bytes); - } - - // Bulk decode fallback: loads entire file into memory and uses decodeAudioData. - private async bulkDecodeFromUrl(url: string, sampleRate: number): Promise { - try { - const source = await resolveMediaElementSource(url); - try { - const response = await fetch(source.src); - const arrayBuffer = await response.arrayBuffer(); - const tempCtx = new OfflineAudioContext(2, 1, sampleRate); - return await tempCtx.decodeAudioData(arrayBuffer); - } finally { - source.revoke(); - } - } catch (error) { - console.warn("[AudioProcessor] Failed to decode audio from URL:", url, error); - return null; - } - } - - // Get the duration of a media file by loading only its metadata. - private async getMediaDurationSec(url: string): Promise { - const source = await resolveMediaElementSource(url); - try { - const media = document.createElement("video"); - media.preload = "metadata"; - media.src = source.src; - - return await new Promise((resolve, reject) => { - const timeout = setTimeout(() => { - cleanup(); - media.src = ""; - media.load(); - reject(new Error("Timed out getting media duration (30s)")); - }, 30_000); - - const onLoaded = () => { - cleanup(); - const duration = media.duration; - media.src = ""; - media.load(); - resolve(Number.isFinite(duration) ? duration : 0); - }; - const onError = () => { - cleanup(); - media.src = ""; - media.load(); - reject(new Error("Failed to get media duration")); - }; - const cleanup = () => { - clearTimeout(timeout); - media.removeEventListener("loadedmetadata", onLoaded); - media.removeEventListener("error", onError); - }; - - media.addEventListener("loadedmetadata", onLoaded); - media.addEventListener("error", onError, { once: true }); - }); - } finally { - source.revoke(); - } - } - - // Build non-overlapping timeline slices from the source timeline, excluding - // trimmed regions and tagging each slice with its playback speed. - private buildTimelineSlices( - sourceDurationMs: number, - trimRegions: TrimLikeRegion[], - speedRegions: SpeedRegion[], - ): TimelineSlice[] { - const boundaries = new Set(); - boundaries.add(0); - boundaries.add(sourceDurationMs); - - for (const trim of trimRegions) { - if (trim.startMs >= 0 && trim.startMs <= sourceDurationMs) boundaries.add(trim.startMs); - if (trim.endMs >= 0 && trim.endMs <= sourceDurationMs) boundaries.add(trim.endMs); - } - for (const speed of speedRegions) { - if (speed.startMs >= 0 && speed.startMs <= sourceDurationMs) - boundaries.add(speed.startMs); - if (speed.endMs >= 0 && speed.endMs <= sourceDurationMs) boundaries.add(speed.endMs); - } - - const sorted = [...boundaries].sort((a, b) => a - b); - const slices: TimelineSlice[] = []; - - for (let i = 0; i < sorted.length - 1; i++) { - const start = sorted[i]; - const end = sorted[i + 1]; - if (end - start < 0.001) continue; - - // Skip segments entirely inside a trim region - const midpoint = (start + end) / 2; - if (this.isInTrimRegion(midpoint, trimRegions)) continue; - - const speedRegion = speedRegions.find( - (s) => midpoint >= s.startMs && midpoint < s.endMs, - ); - - slices.push({ - sourceStartMs: start, - sourceEndMs: end, - speed: speedRegion?.speed ?? 1, - }); - } - - return slices; - } - - // Map a source-timeline timestamp to the corresponding output-timeline timestamp. - private sourceTimeToOutputTime(sourceMs: number, slices: TimelineSlice[]): number { - let outputMs = 0; - - for (const slice of slices) { - if (sourceMs <= slice.sourceStartMs) { - return outputMs; - } - const sliceDurationMs = slice.sourceEndMs - slice.sourceStartMs; - if (sourceMs >= slice.sourceEndMs) { - outputMs += sliceDurationMs / slice.speed; - continue; - } - // Source time falls within this slice - outputMs += (sourceMs - slice.sourceStartMs) / slice.speed; - return outputMs; - } - - return outputMs; - } - - // Schedule an AudioBuffer through the timeline slices in an OfflineAudioContext. - // Each non-trimmed segment creates an AudioBufferSourceNode with the appropriate - // playbackRate for speed regions. When chunkOutputStartSec/chunkDurationSec are - // provided, only sources overlapping the chunk window are scheduled. - private scheduleBufferThroughTimeline( - ctx: OfflineAudioContext, - buffer: AudioBuffer, - slices: TimelineSlice[], - sourceStartDelaySec: number, - gain = 1, - chunkOutputStartSec = 0, - chunkDurationSec = Number.POSITIVE_INFINITY, - mutedOutputRangesSec: Array<{ startSec: number; endSec: number }> = [], - ): void { - let outputOffsetSec = 0; - - for (const slice of slices) { - const sliceSourceDurationSec = (slice.sourceEndMs - slice.sourceStartMs) / 1000; - const sliceOutputDurationSec = sliceSourceDurationSec / slice.speed; - - // Where in the buffer does this slice read from? - const bufferOffsetSec = slice.sourceStartMs / 1000 - sourceStartDelaySec; - - // Skip if slice doesn't overlap with the buffer at all - if ( - bufferOffsetSec + sliceSourceDurationSec <= 0 || - bufferOffsetSec >= buffer.duration - ) { - outputOffsetSec += sliceOutputDurationSec; - continue; - } - - // Clamp to buffer bounds - let effectiveBufferStartSec = Math.max(0, bufferOffsetSec); - const trimmedFromStartSec = effectiveBufferStartSec - bufferOffsetSec; - let effectiveSourceDurationSec = Math.min( - sliceSourceDurationSec - trimmedFromStartSec, - buffer.duration - effectiveBufferStartSec, - ); - - if (effectiveSourceDurationSec <= 0.001) { - outputOffsetSec += sliceOutputDurationSec; - continue; - } - - // Calculate output position (global then chunk-local) - let localOutputStartSec = - outputOffsetSec + trimmedFromStartSec / slice.speed - chunkOutputStartSec; - let localOutputEndSec = localOutputStartSec + effectiveSourceDurationSec / slice.speed; - - // Skip if entirely outside chunk window - if (localOutputEndSec <= 0 || localOutputStartSec >= chunkDurationSec) { - outputOffsetSec += sliceOutputDurationSec; - continue; - } - - // Clip to chunk start - if (localOutputStartSec < 0) { - const skipOutputSec = -localOutputStartSec; - const skipSourceSec = skipOutputSec * slice.speed; - effectiveBufferStartSec += skipSourceSec; - effectiveSourceDurationSec -= skipSourceSec; - localOutputStartSec = 0; - } - - // Clip to chunk end - if (localOutputEndSec > chunkDurationSec) { - const excessOutputSec = localOutputEndSec - chunkDurationSec; - effectiveSourceDurationSec -= excessOutputSec * slice.speed; - } - - if (effectiveSourceDurationSec <= 0.001) { - outputOffsetSec += sliceOutputDurationSec; - continue; - } - - const audibleRanges: Array<{ startSec: number; endSec: number }> = [ - { - startSec: localOutputStartSec + chunkOutputStartSec, - endSec: - localOutputStartSec + - chunkOutputStartSec + - effectiveSourceDurationSec / slice.speed, - }, - ]; - for (const mutedRange of mutedOutputRangesSec) { - for (let rangeIndex = audibleRanges.length - 1; rangeIndex >= 0; rangeIndex -= 1) { - const current = audibleRanges[rangeIndex]; - const overlapStart = Math.max(current.startSec, mutedRange.startSec); - const overlapEnd = Math.min(current.endSec, mutedRange.endSec); - if (overlapEnd <= overlapStart) { - continue; - } - audibleRanges.splice(rangeIndex, 1); - if (current.startSec < overlapStart) { - audibleRanges.push({ startSec: current.startSec, endSec: overlapStart }); - } - if (overlapEnd < current.endSec) { - audibleRanges.push({ startSec: overlapEnd, endSec: current.endSec }); - } - } - } - - for (const audibleRange of audibleRanges) { - const audibleDurationSec = audibleRange.endSec - audibleRange.startSec; - if (audibleDurationSec <= 0.001) { - continue; - } - const source = ctx.createBufferSource(); - const gainNode = ctx.createGain(); - gainNode.gain.value = Math.max(0, Math.min(2, gain)); - - const sourceOffsetSec = - effectiveBufferStartSec + - (audibleRange.startSec - (localOutputStartSec + chunkOutputStartSec)) * - slice.speed; - const localStartSec = audibleRange.startSec - chunkOutputStartSec; - const sourceDurationSec = audibleDurationSec * slice.speed; - - const stretchedBuffer = this.stretchAudioBuffer( - buffer, - slice.speed, - sourceOffsetSec, - sourceDurationSec, - audibleDurationSec, - ctx, - ); - - source.buffer = stretchedBuffer; - source.playbackRate.value = 1; - source.connect(gainNode); - gainNode.connect(ctx.destination); - - source.start(localStartSec); - } - - outputOffsetSec += sliceOutputDurationSec; - } - } - - private stretchAudioBuffer( - originalBuffer: AudioBuffer, - speed: number, - sourceOffsetSec: number, - sourceDurationSec: number, - audibleDurationSec: number, - ctx: BaseAudioContext, - ): AudioBuffer { - const sampleRate = originalBuffer.sampleRate; - const channels = originalBuffer.numberOfChannels; - - const startSample = Math.max(0, Math.floor(sourceOffsetSec * sampleRate)); - const sourceSamples = Math.floor(sourceDurationSec * sampleRate); - const endSample = Math.min(originalBuffer.length, startSample + sourceSamples); - - const outSamples = Math.floor(audibleDurationSec * sampleRate); - if (outSamples <= 0 || startSample >= originalBuffer.length) { - return ctx.createBuffer(channels, 1, sampleRate); - } - - const outBuffer = ctx.createBuffer(channels, outSamples, sampleRate); - - if (Math.abs(speed - 1) < 0.001) { - const copyLength = Math.min(endSample - startSample, outSamples); - if (copyLength > 0) { - for (let c = 0; c < channels; c++) { - outBuffer.copyToChannel( - originalBuffer - .getChannelData(c) - .subarray(startSample, startSample + copyLength), - c, - ); - } - } - return outBuffer; - } - - // WSOLA uses windowing which causes fade-in at the start and fade-out at the end. - // To avoid clicks at chunk boundaries, we render with 100ms of padding and trim it. - const paddingSec = 0.1; - const paddingOutSamples = Math.floor(sampleRate * paddingSec); - const paddingInSamples = Math.floor(paddingOutSamples * speed); - - const workStartIn = Math.max(0, startSample - paddingInSamples); - const workEndIn = Math.min(originalBuffer.length, endSample + paddingInSamples); - - const actualPaddingInStart = startSample - workStartIn; - // We expect the output offset for the requested start to be roughly: - const actualPaddingOutStart = Math.floor(actualPaddingInStart / speed); - - const windowSize = Math.floor(sampleRate * 0.04); - const hopOut = Math.floor(windowSize * 0.5); - const hopIn = Math.floor(hopOut * speed); - const searchRange = Math.floor(sampleRate * 0.015); - - const workOutSamples = Math.floor((workEndIn - workStartIn) / speed) + windowSize * 2; - const workOutBuffer = ctx.createBuffer(channels, workOutSamples, sampleRate); - - const inDataByChannel = Array.from({ length: channels }, (_, c) => - originalBuffer.getChannelData(c), - ); - const workOutDataByChannel = Array.from({ length: channels }, (_, c) => - workOutBuffer.getChannelData(c), - ); - - const window = new Float32Array(windowSize); - for (let i = 0; i < windowSize; i++) { - window[i] = 0.5 * (1 - Math.cos((2 * Math.PI * i) / (windowSize - 1))); - } - - let inOffset = workStartIn; - let outOffset = 0; - - // Initial window - for (let i = 0; i < windowSize; i++) { - if (inOffset + i < workEndIn && outOffset + i < workOutSamples) { - for (let c = 0; c < channels; c++) { - workOutDataByChannel[c][outOffset + i] += - inDataByChannel[c][inOffset + i] * window[i]; - } - } - } - - outOffset += hopOut; - inOffset += hopIn; - - while (outOffset + windowSize < workOutSamples && inOffset + windowSize < workEndIn) { - let bestOffset = inOffset; - const minSearch = Math.max(workStartIn, inOffset - searchRange); - const maxSearch = Math.min(workEndIn - windowSize, inOffset + searchRange); - - if (maxSearch > minSearch) { - let maxCorr = -Infinity; - let bestDelta = 0; - - for (let testOffset = minSearch; testOffset <= maxSearch; testOffset += 4) { - let corr = 0; - for (let i = 0; i < hopOut; i += 4) { - if (outOffset + i < workOutSamples && testOffset + i < workEndIn) { - for (let c = 0; c < channels; c++) { - corr += - workOutDataByChannel[c][outOffset + i] * - inDataByChannel[c][testOffset + i]; - } - } - } - if (corr > maxCorr) { - maxCorr = corr; - bestDelta = testOffset - inOffset; - } - } - bestOffset = inOffset + bestDelta; - } - - for (let i = 0; i < windowSize; i++) { - if (bestOffset + i < workEndIn && outOffset + i < workOutSamples) { - for (let c = 0; c < channels; c++) { - workOutDataByChannel[c][outOffset + i] += - inDataByChannel[c][bestOffset + i] * window[i]; - } - } - } - - outOffset += hopOut; - inOffset += hopIn; - } - - // Transfer the stable middle portion to the final buffer - for (let c = 0; c < channels; c++) { - const finalData = outBuffer.getChannelData(c); - const tempData = workOutBuffer.getChannelData(c); - for (let i = 0; i < outSamples; i++) { - const srcIdx = actualPaddingOutStart + i; - if (srcIdx < workOutSamples) { - finalData[i] = tempData[srcIdx]; - } - } - } - - return outBuffer; - } - - // Create a WAV file header for the given audio parameters. - private createWavHeader( - sampleRate: number, - numChannels: number, - totalFrames: number, - ): ArrayBuffer { - const bytesPerSample = 2; // 16-bit PCM - const dataSize = totalFrames * numChannels * bytesPerSample; - const headerSize = 44; - const header = new ArrayBuffer(headerSize); - const view = new DataView(header); - - const writeString = (offset: number, str: string) => { - for (let i = 0; i < str.length; i++) { - view.setUint8(offset + i, str.charCodeAt(i)); - } - }; - - writeString(0, "RIFF"); - view.setUint32(4, headerSize - 8 + dataSize, true); - writeString(8, "WAVE"); - writeString(12, "fmt "); - view.setUint32(16, 16, true); - view.setUint16(20, 1, true); // PCM format - view.setUint16(22, numChannels, true); - view.setUint32(24, sampleRate, true); - view.setUint32(28, sampleRate * numChannels * bytesPerSample, true); - view.setUint16(32, numChannels * bytesPerSample, true); - view.setUint16(34, bytesPerSample * 8, true); - writeString(36, "data"); - view.setUint32(40, dataSize, true); - - return header; - } - - // Convert an AudioBuffer to chunked 16-bit PCM ArrayBuffers. - // Returns small (~256KB) pieces instead of one massive allocation. - private audioBufferToPcmParts(buffer: AudioBuffer): ArrayBuffer[] { - const PCM_CHUNK_FRAMES = 65536; - const numChannels = buffer.numberOfChannels; - const numFrames = buffer.length; - const bytesPerSample = 2; - const parts: ArrayBuffer[] = []; - - const channels: Float32Array[] = []; - for (let ch = 0; ch < numChannels; ch++) { - channels.push(buffer.getChannelData(ch)); - } - - for (let frameOffset = 0; frameOffset < numFrames; frameOffset += PCM_CHUNK_FRAMES) { - const chunkFrames = Math.min(PCM_CHUNK_FRAMES, numFrames - frameOffset); - const chunkBuffer = new ArrayBuffer(chunkFrames * numChannels * bytesPerSample); - const view = new DataView(chunkBuffer); - - let byteOffset = 0; - for (let i = 0; i < chunkFrames; i++) { - for (let ch = 0; ch < numChannels; ch++) { - const sample = Math.max(-1, Math.min(1, channels[ch][frameOffset + i])); - view.setInt16(byteOffset, sample < 0 ? sample * 0x8000 : sample * 0x7fff, true); - byteOffset += 2; - } - } - - parts.push(chunkBuffer); - } - - return parts; - } - - // Loads a sidecar audio file into a WebDemuxer for direct transcoding (avoiding real-time rendering). - private async loadAudioFileDemuxer(audioPath: string): Promise { - try { - const source = await resolveMediaElementSource(audioPath); - try { - const wasmUrl = new URL("./wasm/web-demuxer.wasm", window.location.href).href; - const demuxer = new WebDemuxer({ wasmFilePath: wasmUrl }); - await demuxer.load(source.src); - return demuxer; - } finally { - source.revoke(); - } - } catch (error) { - console.warn("[AudioProcessor] Failed to create demuxer for sidecar audio:", error); - return null; - } - } - - private cloneWithTimestamp(src: AudioData, newTimestamp: number): AudioData { - const isPlanar = src.format?.includes("planar") ?? false; - const numPlanes = isPlanar ? src.numberOfChannels : 1; - - let totalSize = 0; - for (let planeIndex = 0; planeIndex < numPlanes; planeIndex++) { - totalSize += src.allocationSize({ planeIndex }); - } - - const buffer = new ArrayBuffer(totalSize); - let offset = 0; - - for (let planeIndex = 0; planeIndex < numPlanes; planeIndex++) { - const planeSize = src.allocationSize({ planeIndex }); - src.copyTo(new Uint8Array(buffer, offset, planeSize), { planeIndex }); - offset += planeSize; - } - - return new AudioData({ - format: src.format!, - sampleRate: src.sampleRate, - numberOfFrames: src.numberOfFrames, - numberOfChannels: src.numberOfChannels, - timestamp: newTimestamp, - data: buffer, - }); - } - - private cloneEncodedAudioChunkWithTimestamp( - src: EncodedAudioChunk, - newTimestamp: number, - ): EncodedAudioChunk { - const data = new Uint8Array(src.byteLength); - src.copyTo(data); - - return new EncodedAudioChunk({ - type: src.type, - timestamp: newTimestamp, - duration: src.duration ?? undefined, - data, - }); - } - - private isInTrimRegion(timestampMs: number, trims: TrimLikeRegion[]) { - return trims.some((trim) => timestampMs >= trim.startMs && timestampMs < trim.endMs); - } - - private computeTrimOffset(timestampMs: number, trims: TrimLikeRegion[]) { - let offset = 0; - for (const trim of trims) { - if (trim.endMs <= timestampMs) { - offset += trim.endMs - trim.startMs; - } - } - return offset; - } - - cancel() { - this.cancelled = true; - } } diff --git a/src/lib/exporter/audioMediaProcessor.ts b/src/lib/exporter/audioMediaProcessor.ts new file mode 100644 index 00000000..3359873a --- /dev/null +++ b/src/lib/exporter/audioMediaProcessor.ts @@ -0,0 +1,282 @@ +import { WebDemuxer } from "web-demuxer"; +import { AudioTimelineProcessor } from "./audioTimelineProcessor"; +import { DECODE_BACKPRESSURE_LIMIT, OFFLINE_AUDIO_SAMPLE_RATE } from "./audioProcessorShared"; +import { resolveMediaElementSource } from "./localMediaSource"; + +export class AudioMediaProcessor extends AudioTimelineProcessor { + protected async decodeAudioFromUrl(url: string): Promise { + try { + const buffer = await this.streamDecodeFromUrl(url); + if (buffer) return buffer; + } catch (error) { + console.warn( + "[AudioProcessor] Streaming decode failed, falling back to bulk decode:", + url, + error, + ); + } + return this.bulkDecodeFromUrl(url, OFFLINE_AUDIO_SAMPLE_RATE); + } + + // Streaming decode via WebDemuxer + AudioDecoder. Decodes audio chunk-by-chunk + // without loading the entire compressed file into a contiguous ArrayBuffer. + protected async streamDecodeFromUrl(url: string): Promise { + const source = await resolveMediaElementSource(url); + let demuxer: WebDemuxer | null = null; + + try { + const wasmUrl = new URL("./wasm/web-demuxer.wasm", window.location.href).href; + demuxer = new WebDemuxer({ wasmFilePath: wasmUrl }); + await demuxer.load(source.src); + + let audioConfig: AudioDecoderConfig; + try { + audioConfig = (await demuxer.getDecoderConfig("audio")) as AudioDecoderConfig; + } catch { + return null; // No audio track + } + + const sampleRate = audioConfig.sampleRate || 48_000; + const numChannels = Math.min(audioConfig.numberOfChannels || 2, 2); + + // Accumulate decoded PCM per channel + const channelChunks: Float32Array[][] = Array.from({ length: numChannels }, () => []); + let totalFrames = 0; + let decodeError: Error | null = null; + const decodeCapacityWaiters = new Set<() => void>(); + + const notifyDecodeCapacityAvailable = () => { + if (decodeCapacityWaiters.size === 0) { + return; + } + + const waiters = [...decodeCapacityWaiters]; + decodeCapacityWaiters.clear(); + for (const resolve of waiters) { + resolve(); + } + }; + + const waitForDecodeCapacity = () => + new Promise((resolve) => { + decodeCapacityWaiters.add(resolve); + }); + + const decoder = new AudioDecoder({ + output: (data: AudioData) => { + try { + const frames = data.numberOfFrames; + const dataChannels = Math.min(data.numberOfChannels, numChannels); + const format = data.format; + + if (format?.includes("planar")) { + for (let ch = 0; ch < dataChannels; ch++) { + const size = data.allocationSize({ + planeIndex: ch, + }); + const bytes = new ArrayBuffer(size); + data.copyTo(bytes, { planeIndex: ch }); + channelChunks[ch].push(this.rawToFloat32(bytes, format, frames)); + } + } else if (format) { + // Interleaved format — deinterleave into per-channel arrays. + // Use data.numberOfChannels as stride (not capped dataChannels) + // since the raw buffer contains all source channels. + const srcChannels = data.numberOfChannels; + const size = data.allocationSize({ planeIndex: 0 }); + const bytes = new ArrayBuffer(size); + data.copyTo(bytes, { planeIndex: 0 }); + const interleaved = this.rawToFloat32( + bytes, + format, + frames * srcChannels, + ); + for (let ch = 0; ch < dataChannels; ch++) { + const chData = new Float32Array(frames); + for (let i = 0; i < frames; i++) { + chData[i] = interleaved[i * srcChannels + ch]; + } + channelChunks[ch].push(chData); + } + } + + // Fill missing channels with silence + for (let ch = dataChannels; ch < numChannels; ch++) { + channelChunks[ch].push(new Float32Array(frames)); + } + + totalFrames += frames; + } finally { + data.close(); + notifyDecodeCapacityAvailable(); + } + }, + error: (err: DOMException) => { + decodeError = new Error(`Streaming audio decode error: ${err.message}`); + notifyDecodeCapacityAvailable(); + }, + }); + + decoder.configure(audioConfig); + + const audioStream = demuxer.read("audio"); + const reader = (audioStream as ReadableStream).getReader(); + + try { + while (!this.cancelled) { + if (decodeError) throw decodeError; + const { done, value: chunk } = await reader.read(); + if (done || !chunk) break; + + decoder.decode(chunk); + + while (decoder.decodeQueueSize > DECODE_BACKPRESSURE_LIMIT && !this.cancelled) { + if (decodeError) throw decodeError; + await waitForDecodeCapacity(); + } + } + + if (decoder.state === "configured") { + await decoder.flush(); + } + if (decodeError) throw decodeError; + } finally { + notifyDecodeCapacityAvailable(); + try { + await reader.cancel(); + } catch { + /* reader already closed */ + } + if (decoder.state === "configured") { + decoder.close(); + } + } + + if (totalFrames === 0) return null; + + // Build AudioBuffer from accumulated chunks + const audioBuffer = new AudioBuffer({ + length: totalFrames, + numberOfChannels: numChannels, + sampleRate, + }); + for (let ch = 0; ch < numChannels; ch++) { + const channelData = audioBuffer.getChannelData(ch); + let writeOffset = 0; + for (const chunk of channelChunks[ch]) { + channelData.set(chunk, writeOffset); + writeOffset += chunk.length; + } + } + + return audioBuffer; + } finally { + source.revoke(); + try { + demuxer?.destroy(); + } catch { + /* cleanup */ + } + } + } + + // Convert raw bytes from AudioData to Float32Array based on the sample format. + protected rawToFloat32(bytes: ArrayBuffer, format: string, sampleCount: number): Float32Array { + if (format.startsWith("f32")) { + return new Float32Array(bytes); + } + if (format.startsWith("s16")) { + const int16 = new Int16Array(bytes); + const f32 = new Float32Array(sampleCount); + for (let i = 0; i < sampleCount; i++) { + f32[i] = int16[i] / 0x8000; + } + return f32; + } + if (format.startsWith("s32")) { + const int32 = new Int32Array(bytes); + const f32 = new Float32Array(sampleCount); + for (let i = 0; i < sampleCount; i++) { + f32[i] = int32[i] / 0x80000000; + } + return f32; + } + if (format.startsWith("u8")) { + const uint8 = new Uint8Array(bytes); + const f32 = new Float32Array(sampleCount); + for (let i = 0; i < sampleCount; i++) { + f32[i] = (uint8[i] - 128) / 128; + } + return f32; + } + // Unknown format — attempt float32 interpretation + return new Float32Array(bytes); + } + + // Bulk decode fallback: loads entire file into memory and uses decodeAudioData. + protected async bulkDecodeFromUrl( + url: string, + sampleRate: number, + ): Promise { + try { + const source = await resolveMediaElementSource(url); + try { + const response = await fetch(source.src); + const arrayBuffer = await response.arrayBuffer(); + const tempCtx = new OfflineAudioContext(2, 1, sampleRate); + return await tempCtx.decodeAudioData(arrayBuffer); + } finally { + source.revoke(); + } + } catch (error) { + console.warn("[AudioProcessor] Failed to decode audio from URL:", url, error); + return null; + } + } + + // Get the duration of a media file by loading only its metadata. + protected async getMediaDurationSec(url: string): Promise { + const source = await resolveMediaElementSource(url); + try { + const media = document.createElement("video"); + media.preload = "metadata"; + media.src = source.src; + + return await new Promise((resolve, reject) => { + const timeout = setTimeout(() => { + cleanup(); + media.src = ""; + media.load(); + reject(new Error("Timed out getting media duration (30s)")); + }, 30_000); + + const onLoaded = () => { + cleanup(); + const duration = media.duration; + media.src = ""; + media.load(); + resolve(Number.isFinite(duration) ? duration : 0); + }; + const onError = () => { + cleanup(); + media.src = ""; + media.load(); + reject(new Error("Failed to get media duration")); + }; + const cleanup = () => { + clearTimeout(timeout); + media.removeEventListener("loadedmetadata", onLoaded); + media.removeEventListener("error", onError); + }; + + media.addEventListener("loadedmetadata", onLoaded); + media.addEventListener("error", onError, { once: true }); + }); + } finally { + source.revoke(); + } + } + + // Build non-overlapping timeline slices from the source timeline, excluding + // trimmed regions and tagging each slice with its playback speed. +} diff --git a/src/lib/exporter/audioProcessorBase.ts b/src/lib/exporter/audioProcessorBase.ts new file mode 100644 index 00000000..07842504 --- /dev/null +++ b/src/lib/exporter/audioProcessorBase.ts @@ -0,0 +1,154 @@ +import { WebDemuxer } from "web-demuxer"; +import { resolveMediaElementSource } from "./localMediaSource"; +import type { TrimLikeRegion } from "./audioProcessorShared"; + +export class AudioProcessorBase { + protected cancelled = false; + protected onProgress?: (progress: number) => void; + protected createWavHeader( + sampleRate: number, + numChannels: number, + totalFrames: number, + ): ArrayBuffer { + const bytesPerSample = 2; // 16-bit PCM + const dataSize = totalFrames * numChannels * bytesPerSample; + const headerSize = 44; + const header = new ArrayBuffer(headerSize); + const view = new DataView(header); + + const writeString = (offset: number, str: string) => { + for (let i = 0; i < str.length; i++) { + view.setUint8(offset + i, str.charCodeAt(i)); + } + }; + + writeString(0, "RIFF"); + view.setUint32(4, headerSize - 8 + dataSize, true); + writeString(8, "WAVE"); + writeString(12, "fmt "); + view.setUint32(16, 16, true); + view.setUint16(20, 1, true); // PCM format + view.setUint16(22, numChannels, true); + view.setUint32(24, sampleRate, true); + view.setUint32(28, sampleRate * numChannels * bytesPerSample, true); + view.setUint16(32, numChannels * bytesPerSample, true); + view.setUint16(34, bytesPerSample * 8, true); + writeString(36, "data"); + view.setUint32(40, dataSize, true); + + return header; + } + + // Convert an AudioBuffer to chunked 16-bit PCM ArrayBuffers. + // Returns small (~256KB) pieces instead of one massive allocation. + protected audioBufferToPcmParts(buffer: AudioBuffer): ArrayBuffer[] { + const PCM_CHUNK_FRAMES = 65536; + const numChannels = buffer.numberOfChannels; + const numFrames = buffer.length; + const bytesPerSample = 2; + const parts: ArrayBuffer[] = []; + + const channels: Float32Array[] = []; + for (let ch = 0; ch < numChannels; ch++) { + channels.push(buffer.getChannelData(ch)); + } + + for (let frameOffset = 0; frameOffset < numFrames; frameOffset += PCM_CHUNK_FRAMES) { + const chunkFrames = Math.min(PCM_CHUNK_FRAMES, numFrames - frameOffset); + const chunkBuffer = new ArrayBuffer(chunkFrames * numChannels * bytesPerSample); + const view = new DataView(chunkBuffer); + + let byteOffset = 0; + for (let i = 0; i < chunkFrames; i++) { + for (let ch = 0; ch < numChannels; ch++) { + const sample = Math.max(-1, Math.min(1, channels[ch][frameOffset + i])); + view.setInt16(byteOffset, sample < 0 ? sample * 0x8000 : sample * 0x7fff, true); + byteOffset += 2; + } + } + + parts.push(chunkBuffer); + } + + return parts; + } + + // Loads a sidecar audio file into a WebDemuxer for direct transcoding (avoiding real-time rendering). + protected async loadAudioFileDemuxer(audioPath: string): Promise { + try { + const source = await resolveMediaElementSource(audioPath); + try { + const wasmUrl = new URL("./wasm/web-demuxer.wasm", window.location.href).href; + const demuxer = new WebDemuxer({ wasmFilePath: wasmUrl }); + await demuxer.load(source.src); + return demuxer; + } finally { + source.revoke(); + } + } catch (error) { + console.warn("[AudioProcessor] Failed to create demuxer for sidecar audio:", error); + return null; + } + } + + protected cloneWithTimestamp(src: AudioData, newTimestamp: number): AudioData { + const isPlanar = src.format?.includes("planar") ?? false; + const numPlanes = isPlanar ? src.numberOfChannels : 1; + + let totalSize = 0; + for (let planeIndex = 0; planeIndex < numPlanes; planeIndex++) { + totalSize += src.allocationSize({ planeIndex }); + } + + const buffer = new ArrayBuffer(totalSize); + let offset = 0; + + for (let planeIndex = 0; planeIndex < numPlanes; planeIndex++) { + const planeSize = src.allocationSize({ planeIndex }); + src.copyTo(new Uint8Array(buffer, offset, planeSize), { planeIndex }); + offset += planeSize; + } + + return new AudioData({ + format: src.format!, + sampleRate: src.sampleRate, + numberOfFrames: src.numberOfFrames, + numberOfChannels: src.numberOfChannels, + timestamp: newTimestamp, + data: buffer, + }); + } + + protected cloneEncodedAudioChunkWithTimestamp( + src: EncodedAudioChunk, + newTimestamp: number, + ): EncodedAudioChunk { + const data = new Uint8Array(src.byteLength); + src.copyTo(data); + + return new EncodedAudioChunk({ + type: src.type, + timestamp: newTimestamp, + duration: src.duration ?? undefined, + data, + }); + } + + protected isInTrimRegion(timestampMs: number, trims: TrimLikeRegion[]) { + return trims.some((trim) => timestampMs >= trim.startMs && timestampMs < trim.endMs); + } + + protected computeTrimOffset(timestampMs: number, trims: TrimLikeRegion[]) { + let offset = 0; + for (const trim of trims) { + if (trim.endMs <= timestampMs) { + offset += trim.endMs - trim.startMs; + } + } + return offset; + } + + cancel() { + this.cancelled = true; + } +} diff --git a/src/lib/exporter/audioProcessorShared.ts b/src/lib/exporter/audioProcessorShared.ts new file mode 100644 index 00000000..ca6e35e9 --- /dev/null +++ b/src/lib/exporter/audioProcessorShared.ts @@ -0,0 +1,144 @@ +import { SOURCE_AUDIO_NORMALIZE_GAIN } from "@/components/video-editor/audio/audioTypes"; +import type { + AudioRegion, + ClipRegion, + SourceAudioTrackSettings, + TrimRegion, +} from "@/components/video-editor/types"; +import type { SourceTrackId } from "@/lib/exporter/audioRoutingEngine"; + +export const AUDIO_BITRATE = 128_000; +export const DECODE_BACKPRESSURE_LIMIT = 20; +export const ENCODE_BACKPRESSURE_LIMIT = 20; +export const MIN_SPEED_REGION_DELTA_MS = 0.0001; +export const MP4_AUDIO_CODEC = "mp4a.40.2"; +export const OFFLINE_AUDIO_SAMPLE_RATE = 48_000; +export const OFFLINE_ENCODE_CHUNK_FRAMES = 1024; +export const OFFLINE_CHUNK_DURATION_SEC = 30; +const OFFLINE_MIX_SOFT_LIMITER_THRESHOLD = 0.9; +const OFFLINE_MIX_SOFT_LIMITER_CEILING = 0.985; + +function softLimitSample(sample: number): number { + const magnitude = Math.abs(sample); + if (magnitude <= OFFLINE_MIX_SOFT_LIMITER_THRESHOLD) { + return sample; + } + + const sign = sample < 0 ? -1 : 1; + const kneeRange = 1 - OFFLINE_MIX_SOFT_LIMITER_THRESHOLD; + const limitedMagnitude = + OFFLINE_MIX_SOFT_LIMITER_THRESHOLD + + kneeRange * Math.tanh((magnitude - OFFLINE_MIX_SOFT_LIMITER_THRESHOLD) / kneeRange); + return sign * Math.min(OFFLINE_MIX_SOFT_LIMITER_CEILING, limitedMagnitude); +} + +export function softLimitOfflineMixPeaksInPlace(buffer: AudioBuffer): boolean { + let changed = false; + for (let channel = 0; channel < buffer.numberOfChannels; channel += 1) { + const data = buffer.getChannelData(channel); + for (let index = 0; index < data.length; index += 1) { + const sample = data[index]; + if (!Number.isFinite(sample)) { + data[index] = 0; + changed = true; + continue; + } + + const limited = softLimitSample(sample); + if (limited !== sample) { + data[index] = limited; + changed = true; + } + } + } + return changed; +} + +export function resolveSourceTrackGain( + sourceAudioTrackSettings: SourceAudioTrackSettings | undefined, + trackId: "mic" | "system" | "mixed", +) { + const settings = sourceAudioTrackSettings?.[trackId]; + if (!settings) { + return 1; + } + const normalizeGain = settings.normalize ? SOURCE_AUDIO_NORMALIZE_GAIN : 1; + return Math.max(0, Math.min(2, settings.volume * normalizeGain)); +} + +export function getSourceTrackIdFromPath(audioPath: string): SourceTrackId { + const normalized = audioPath.toLowerCase(); + // Check for common patterns like .mic., -mic., mic.mp4, etc. + if ( + normalized.includes(".mic.") || + normalized.includes("-mic.") || + normalized.includes("_mic_") || + normalized.includes("/mic.") || + normalized.includes("\\mic.") || + normalized.endsWith("mic.mp4") || + normalized.endsWith("mic.m4a") || + normalized.endsWith("mic.wav") + ) { + return "mic"; + } + if ( + normalized.includes(".system.") || + normalized.includes("-system.") || + normalized.includes("_system_") || + normalized.includes("/system.") || + normalized.includes("\\system.") || + normalized.endsWith("system.mp4") || + normalized.endsWith("system.m4a") || + normalized.endsWith("system.wav") + ) { + return "system"; + } + return "mixed"; +} + +export function hasNonDefaultSourceTrackSettings( + sourceAudioTrackSettings?: SourceAudioTrackSettings, +) { + if (!sourceAudioTrackSettings) { + return false; + } + return Object.values(sourceAudioTrackSettings).some( + (settings) => + Math.abs((settings?.volume ?? 1) - 1) > 0.0005 || Boolean(settings?.normalize), + ); +} + +export interface TimelineSlice { + sourceStartMs: number; + sourceEndMs: number; + speed: number; +} + +export interface PreparedOfflineRender { + mainBufferEntry: { buffer: AudioBuffer; gain: number } | null; + companionEntries: Array<{ buffer: AudioBuffer; startDelaySec: number; gain: number }>; + regionEntries: Array<{ buffer: AudioBuffer; region: AudioRegion }>; + mutedSourceOutputRangesSec: Array<{ startSec: number; endSec: number }>; + slices: TimelineSlice[]; + outputDurationMs: number; + numChannels: number; +} + +export async function isAacAudioEncodingSupported( + sampleRate = 48_000, + numberOfChannels = 2, +): Promise { + try { + const support = await AudioEncoder.isConfigSupported({ + codec: MP4_AUDIO_CODEC, + sampleRate, + numberOfChannels, + bitrate: AUDIO_BITRATE, + }); + return support.supported === true; + } catch { + return false; + } +} + +export type TrimLikeRegion = TrimRegion | ClipRegion; diff --git a/src/lib/exporter/audioTimelineProcessor.ts b/src/lib/exporter/audioTimelineProcessor.ts new file mode 100644 index 00000000..44253799 --- /dev/null +++ b/src/lib/exporter/audioTimelineProcessor.ts @@ -0,0 +1,355 @@ +import type { SpeedRegion } from "@/components/video-editor/types"; +import { AudioProcessorBase } from "./audioProcessorBase"; +import type { TimelineSlice, TrimLikeRegion } from "./audioProcessorShared"; + +export class AudioTimelineProcessor extends AudioProcessorBase { + protected buildTimelineSlices( + sourceDurationMs: number, + trimRegions: TrimLikeRegion[], + speedRegions: SpeedRegion[], + ): TimelineSlice[] { + const boundaries = new Set(); + boundaries.add(0); + boundaries.add(sourceDurationMs); + + for (const trim of trimRegions) { + if (trim.startMs >= 0 && trim.startMs <= sourceDurationMs) boundaries.add(trim.startMs); + if (trim.endMs >= 0 && trim.endMs <= sourceDurationMs) boundaries.add(trim.endMs); + } + for (const speed of speedRegions) { + if (speed.startMs >= 0 && speed.startMs <= sourceDurationMs) + boundaries.add(speed.startMs); + if (speed.endMs >= 0 && speed.endMs <= sourceDurationMs) boundaries.add(speed.endMs); + } + + const sorted = [...boundaries].sort((a, b) => a - b); + const slices: TimelineSlice[] = []; + + for (let i = 0; i < sorted.length - 1; i++) { + const start = sorted[i]; + const end = sorted[i + 1]; + if (end - start < 0.001) continue; + + // Skip segments entirely inside a trim region + const midpoint = (start + end) / 2; + if (this.isInTrimRegion(midpoint, trimRegions)) continue; + + const speedRegion = speedRegions.find( + (s) => midpoint >= s.startMs && midpoint < s.endMs, + ); + + slices.push({ + sourceStartMs: start, + sourceEndMs: end, + speed: speedRegion?.speed ?? 1, + }); + } + + return slices; + } + + // Map a source-timeline timestamp to the corresponding output-timeline timestamp. + protected sourceTimeToOutputTime(sourceMs: number, slices: TimelineSlice[]): number { + let outputMs = 0; + + for (const slice of slices) { + if (sourceMs <= slice.sourceStartMs) { + return outputMs; + } + const sliceDurationMs = slice.sourceEndMs - slice.sourceStartMs; + if (sourceMs >= slice.sourceEndMs) { + outputMs += sliceDurationMs / slice.speed; + continue; + } + // Source time falls within this slice + outputMs += (sourceMs - slice.sourceStartMs) / slice.speed; + return outputMs; + } + + return outputMs; + } + + // Schedule an AudioBuffer through the timeline slices in an OfflineAudioContext. + // Each non-trimmed segment creates an AudioBufferSourceNode with the appropriate + // playbackRate for speed regions. When chunkOutputStartSec/chunkDurationSec are + // provided, only sources overlapping the chunk window are scheduled. + protected scheduleBufferThroughTimeline( + ctx: OfflineAudioContext, + buffer: AudioBuffer, + slices: TimelineSlice[], + sourceStartDelaySec: number, + gain = 1, + chunkOutputStartSec = 0, + chunkDurationSec = Number.POSITIVE_INFINITY, + mutedOutputRangesSec: Array<{ startSec: number; endSec: number }> = [], + ): void { + let outputOffsetSec = 0; + + for (const slice of slices) { + const sliceSourceDurationSec = (slice.sourceEndMs - slice.sourceStartMs) / 1000; + const sliceOutputDurationSec = sliceSourceDurationSec / slice.speed; + + // Where in the buffer does this slice read from? + const bufferOffsetSec = slice.sourceStartMs / 1000 - sourceStartDelaySec; + + // Skip if slice doesn't overlap with the buffer at all + if ( + bufferOffsetSec + sliceSourceDurationSec <= 0 || + bufferOffsetSec >= buffer.duration + ) { + outputOffsetSec += sliceOutputDurationSec; + continue; + } + + // Clamp to buffer bounds + let effectiveBufferStartSec = Math.max(0, bufferOffsetSec); + const trimmedFromStartSec = effectiveBufferStartSec - bufferOffsetSec; + let effectiveSourceDurationSec = Math.min( + sliceSourceDurationSec - trimmedFromStartSec, + buffer.duration - effectiveBufferStartSec, + ); + + if (effectiveSourceDurationSec <= 0.001) { + outputOffsetSec += sliceOutputDurationSec; + continue; + } + + // Calculate output position (global then chunk-local) + let localOutputStartSec = + outputOffsetSec + trimmedFromStartSec / slice.speed - chunkOutputStartSec; + let localOutputEndSec = localOutputStartSec + effectiveSourceDurationSec / slice.speed; + + // Skip if entirely outside chunk window + if (localOutputEndSec <= 0 || localOutputStartSec >= chunkDurationSec) { + outputOffsetSec += sliceOutputDurationSec; + continue; + } + + // Clip to chunk start + if (localOutputStartSec < 0) { + const skipOutputSec = -localOutputStartSec; + const skipSourceSec = skipOutputSec * slice.speed; + effectiveBufferStartSec += skipSourceSec; + effectiveSourceDurationSec -= skipSourceSec; + localOutputStartSec = 0; + } + + // Clip to chunk end + if (localOutputEndSec > chunkDurationSec) { + const excessOutputSec = localOutputEndSec - chunkDurationSec; + effectiveSourceDurationSec -= excessOutputSec * slice.speed; + } + + if (effectiveSourceDurationSec <= 0.001) { + outputOffsetSec += sliceOutputDurationSec; + continue; + } + + const audibleRanges: Array<{ startSec: number; endSec: number }> = [ + { + startSec: localOutputStartSec + chunkOutputStartSec, + endSec: + localOutputStartSec + + chunkOutputStartSec + + effectiveSourceDurationSec / slice.speed, + }, + ]; + for (const mutedRange of mutedOutputRangesSec) { + for (let rangeIndex = audibleRanges.length - 1; rangeIndex >= 0; rangeIndex -= 1) { + const current = audibleRanges[rangeIndex]; + const overlapStart = Math.max(current.startSec, mutedRange.startSec); + const overlapEnd = Math.min(current.endSec, mutedRange.endSec); + if (overlapEnd <= overlapStart) { + continue; + } + audibleRanges.splice(rangeIndex, 1); + if (current.startSec < overlapStart) { + audibleRanges.push({ startSec: current.startSec, endSec: overlapStart }); + } + if (overlapEnd < current.endSec) { + audibleRanges.push({ startSec: overlapEnd, endSec: current.endSec }); + } + } + } + + for (const audibleRange of audibleRanges) { + const audibleDurationSec = audibleRange.endSec - audibleRange.startSec; + if (audibleDurationSec <= 0.001) { + continue; + } + const source = ctx.createBufferSource(); + const gainNode = ctx.createGain(); + gainNode.gain.value = Math.max(0, Math.min(2, gain)); + + const sourceOffsetSec = + effectiveBufferStartSec + + (audibleRange.startSec - (localOutputStartSec + chunkOutputStartSec)) * + slice.speed; + const localStartSec = audibleRange.startSec - chunkOutputStartSec; + const sourceDurationSec = audibleDurationSec * slice.speed; + + const stretchedBuffer = this.stretchAudioBuffer( + buffer, + slice.speed, + sourceOffsetSec, + sourceDurationSec, + audibleDurationSec, + ctx, + ); + + source.buffer = stretchedBuffer; + source.playbackRate.value = 1; + source.connect(gainNode); + gainNode.connect(ctx.destination); + + source.start(localStartSec); + } + + outputOffsetSec += sliceOutputDurationSec; + } + } + + protected stretchAudioBuffer( + originalBuffer: AudioBuffer, + speed: number, + sourceOffsetSec: number, + sourceDurationSec: number, + audibleDurationSec: number, + ctx: BaseAudioContext, + ): AudioBuffer { + const sampleRate = originalBuffer.sampleRate; + const channels = originalBuffer.numberOfChannels; + + const startSample = Math.max(0, Math.floor(sourceOffsetSec * sampleRate)); + const sourceSamples = Math.floor(sourceDurationSec * sampleRate); + const endSample = Math.min(originalBuffer.length, startSample + sourceSamples); + + const outSamples = Math.floor(audibleDurationSec * sampleRate); + if (outSamples <= 0 || startSample >= originalBuffer.length) { + return ctx.createBuffer(channels, 1, sampleRate); + } + + const outBuffer = ctx.createBuffer(channels, outSamples, sampleRate); + + if (Math.abs(speed - 1) < 0.001) { + const copyLength = Math.min(endSample - startSample, outSamples); + if (copyLength > 0) { + for (let c = 0; c < channels; c++) { + outBuffer.copyToChannel( + originalBuffer + .getChannelData(c) + .subarray(startSample, startSample + copyLength), + c, + ); + } + } + return outBuffer; + } + + // WSOLA uses windowing which causes fade-in at the start and fade-out at the end. + // To avoid clicks at chunk boundaries, we render with 100ms of padding and trim it. + const paddingSec = 0.1; + const paddingOutSamples = Math.floor(sampleRate * paddingSec); + const paddingInSamples = Math.floor(paddingOutSamples * speed); + + const workStartIn = Math.max(0, startSample - paddingInSamples); + const workEndIn = Math.min(originalBuffer.length, endSample + paddingInSamples); + + const actualPaddingInStart = startSample - workStartIn; + // We expect the output offset for the requested start to be roughly: + const actualPaddingOutStart = Math.floor(actualPaddingInStart / speed); + + const windowSize = Math.floor(sampleRate * 0.04); + const hopOut = Math.floor(windowSize * 0.5); + const hopIn = Math.floor(hopOut * speed); + const searchRange = Math.floor(sampleRate * 0.015); + + const workOutSamples = Math.floor((workEndIn - workStartIn) / speed) + windowSize * 2; + const workOutBuffer = ctx.createBuffer(channels, workOutSamples, sampleRate); + + const inDataByChannel = Array.from({ length: channels }, (_, c) => + originalBuffer.getChannelData(c), + ); + const workOutDataByChannel = Array.from({ length: channels }, (_, c) => + workOutBuffer.getChannelData(c), + ); + + const window = new Float32Array(windowSize); + for (let i = 0; i < windowSize; i++) { + window[i] = 0.5 * (1 - Math.cos((2 * Math.PI * i) / (windowSize - 1))); + } + + let inOffset = workStartIn; + let outOffset = 0; + + // Initial window + for (let i = 0; i < windowSize; i++) { + if (inOffset + i < workEndIn && outOffset + i < workOutSamples) { + for (let c = 0; c < channels; c++) { + workOutDataByChannel[c][outOffset + i] += + inDataByChannel[c][inOffset + i] * window[i]; + } + } + } + + outOffset += hopOut; + inOffset += hopIn; + + while (outOffset + windowSize < workOutSamples && inOffset + windowSize < workEndIn) { + let bestOffset = inOffset; + const minSearch = Math.max(workStartIn, inOffset - searchRange); + const maxSearch = Math.min(workEndIn - windowSize, inOffset + searchRange); + + if (maxSearch > minSearch) { + let maxCorr = -Infinity; + let bestDelta = 0; + + for (let testOffset = minSearch; testOffset <= maxSearch; testOffset += 4) { + let corr = 0; + for (let i = 0; i < hopOut; i += 4) { + if (outOffset + i < workOutSamples && testOffset + i < workEndIn) { + for (let c = 0; c < channels; c++) { + corr += + workOutDataByChannel[c][outOffset + i] * + inDataByChannel[c][testOffset + i]; + } + } + } + if (corr > maxCorr) { + maxCorr = corr; + bestDelta = testOffset - inOffset; + } + } + bestOffset = inOffset + bestDelta; + } + + for (let i = 0; i < windowSize; i++) { + if (bestOffset + i < workEndIn && outOffset + i < workOutSamples) { + for (let c = 0; c < channels; c++) { + workOutDataByChannel[c][outOffset + i] += + inDataByChannel[c][bestOffset + i] * window[i]; + } + } + } + + outOffset += hopOut; + inOffset += hopIn; + } + + // Transfer the stable middle portion to the final buffer + for (let c = 0; c < channels; c++) { + const finalData = outBuffer.getChannelData(c); + const tempData = workOutBuffer.getChannelData(c); + for (let i = 0; i < outSamples; i++) { + const srcIdx = actualPaddingOutStart + i; + if (srcIdx < workOutSamples) { + finalData[i] = tempData[srcIdx]; + } + } + } + + return outBuffer; + } + + // Create a WAV file header for the given audio parameters. +} diff --git a/src/lib/exporter/audioTranscodeProcessor.ts b/src/lib/exporter/audioTranscodeProcessor.ts new file mode 100644 index 00000000..209a280a --- /dev/null +++ b/src/lib/exporter/audioTranscodeProcessor.ts @@ -0,0 +1,275 @@ +import type { WebDemuxer } from "web-demuxer"; +import { OfflineAudioProcessor } from "./offlineAudioProcessor"; +import { + AUDIO_BITRATE, + DECODE_BACKPRESSURE_LIMIT, + ENCODE_BACKPRESSURE_LIMIT, + MP4_AUDIO_CODEC, + type TrimLikeRegion, +} from "./audioProcessorShared"; +import type { VideoMuxer } from "./muxer"; + +export class AudioTranscodeProcessor extends OfflineAudioProcessor { + protected async processTrimOnlyAudio( + demuxer: WebDemuxer, + muxer: VideoMuxer, + sortedTrims: TrimLikeRegion[], + readEndSec?: number, + ): Promise { + let audioConfig: AudioDecoderConfig; + try { + audioConfig = (await demuxer.getDecoderConfig("audio")) as AudioDecoderConfig; + } catch { + console.warn("[AudioProcessor] No audio track found, skipping"); + return; + } + + const codecCheck = await AudioDecoder.isConfigSupported(audioConfig); + if (!codecCheck.supported) { + console.warn("[AudioProcessor] Audio codec not supported:", audioConfig.codec); + return; + } + + const audioStream = + typeof readEndSec === "number" + ? demuxer.read("audio", 0, readEndSec) + : demuxer.read("audio"); + + let sourceTimestampOffsetUs: number | null = null; + + await this.transcodeAudioStream( + audioStream as ReadableStream, + audioConfig, + muxer, + { + observeChunkTimestampUs: (timestampUs) => { + if (sourceTimestampOffsetUs === null) { + sourceTimestampOffsetUs = timestampUs; + } + }, + shouldSkipChunk: (timestampMs) => this.isInTrimRegion(timestampMs, sortedTrims), + transformAudioData: (data) => { + const timestampMs = data.timestamp / 1000; + const trimOffsetMs = this.computeTrimOffset(timestampMs, sortedTrims); + const adjustedTimestampUs = + data.timestamp - (sourceTimestampOffsetUs ?? 0) - trimOffsetMs * 1000; + return this.cloneWithTimestamp(data, Math.max(0, adjustedTimestampUs)); + }, + }, + ); + } + + protected async transcodeAudioStream( + audioStream: ReadableStream, + audioConfig: AudioDecoderConfig, + muxer: VideoMuxer, + options: { + observeChunkTimestampUs?: (timestampUs: number) => void; + shouldSkipChunk?: (timestampMs: number) => boolean; + transformAudioData?: (data: AudioData) => AudioData | null; + } = {}, + ): Promise { + const pendingFrames: AudioData[] = []; + let decodeError: Error | null = null; + let encodeError: Error | null = null; + let muxError: Error | null = null; + let pendingMuxing = Promise.resolve(); + const capacityWaiters = new Set<() => void>(); + + const notifyCapacityAvailable = () => { + if (capacityWaiters.size === 0) { + return; + } + + const waiters = [...capacityWaiters]; + capacityWaiters.clear(); + for (const resolve of waiters) { + resolve(); + } + }; + + const waitForCapacity = () => + new Promise((resolve) => { + capacityWaiters.add(resolve); + }); + + const failIfNeeded = () => { + if (decodeError) throw decodeError; + if (encodeError) throw encodeError; + if (muxError) throw muxError; + }; + + const pumpEncodedFrames = () => { + while (!this.cancelled && pendingFrames.length > 0) { + if (encodeError || muxError) { + break; + } + if (encoder.encodeQueueSize >= ENCODE_BACKPRESSURE_LIMIT) { + break; + } + + const frame = pendingFrames.shift(); + if (!frame) { + break; + } + + encoder.encode(frame); + frame.close(); + notifyCapacityAvailable(); + } + }; + + const cleanupPendingFrames = () => { + for (const frame of pendingFrames) { + frame.close(); + } + pendingFrames.length = 0; + }; + + const sampleRate = audioConfig.sampleRate || 48_000; + const channels = audioConfig.numberOfChannels || 2; + const encodeConfig: AudioEncoderConfig = { + codec: MP4_AUDIO_CODEC, + sampleRate, + numberOfChannels: channels, + bitrate: AUDIO_BITRATE, + }; + + const encodeSupport = await AudioEncoder.isConfigSupported(encodeConfig); + if (!encodeSupport.supported) { + console.warn("[AudioProcessor] AAC encoding not supported, skipping audio"); + return; + } + + const encoder = new AudioEncoder({ + output: (chunk: EncodedAudioChunk, meta?: EncodedAudioChunkMetadata) => { + pendingMuxing = pendingMuxing + .then(async () => { + if (this.cancelled) { + return; + } + await muxer.addAudioChunk(chunk, meta); + }) + .catch((error) => { + muxError = error instanceof Error ? error : new Error(String(error)); + notifyCapacityAvailable(); + }); + notifyCapacityAvailable(); + }, + error: (error: DOMException) => { + encodeError = new Error(`[AudioProcessor] Encode error: ${error.message}`); + notifyCapacityAvailable(); + }, + }); + + encoder.configure(encodeConfig); + + const decoder = new AudioDecoder({ + output: (data: AudioData) => { + if (this.cancelled || encodeError || muxError) { + data.close(); + return; + } + + const transformed = options.transformAudioData + ? options.transformAudioData(data) + : data; + + if (transformed !== data) { + data.close(); + } + + if (!transformed) { + return; + } + + pendingFrames.push(transformed); + notifyCapacityAvailable(); + }, + error: (error: DOMException) => { + decodeError = new Error(`[AudioProcessor] Decode error: ${error.message}`); + notifyCapacityAvailable(); + }, + }); + decoder.configure(audioConfig); + + let reader: ReadableStreamDefaultReader | null = null; + + try { + reader = audioStream.getReader(); + while (!this.cancelled) { + failIfNeeded(); + + const { done, value: chunk } = await reader.read(); + if (done || !chunk) break; + + options.observeChunkTimestampUs?.(chunk.timestamp); + const timestampMs = chunk.timestamp / 1000; + if (options.shouldSkipChunk?.(timestampMs)) continue; + + decoder.decode(chunk); + pumpEncodedFrames(); + + while ( + !this.cancelled && + (decoder.decodeQueueSize > DECODE_BACKPRESSURE_LIMIT || + pendingFrames.length > DECODE_BACKPRESSURE_LIMIT || + encoder.encodeQueueSize >= ENCODE_BACKPRESSURE_LIMIT) + ) { + failIfNeeded(); + pumpEncodedFrames(); + await waitForCapacity(); + } + } + + if (decoder.state === "configured") { + await decoder.flush(); + } + + while (!this.cancelled && (pendingFrames.length > 0 || encoder.encodeQueueSize > 0)) { + failIfNeeded(); + pumpEncodedFrames(); + if (pendingFrames.length > 0 || encoder.encodeQueueSize > 0) { + await waitForCapacity(); + } + } + + failIfNeeded(); + + if (encoder.state === "configured") { + await encoder.flush(); + } + + await pendingMuxing; + failIfNeeded(); + } finally { + notifyCapacityAvailable(); + if (reader) { + try { + await reader.cancel(); + } catch { + // reader already closed + } + } + + cleanupPendingFrames(); + + if (decoder.state === "configured") { + decoder.close(); + } + + if (encoder.state === "configured") { + encoder.close(); + } + } + + if (this.cancelled) { + return; + } + } + + // ---------- Offline audio rendering pipeline ---------- + // Replaces the old real-time MediaElement+MediaRecorder approach with + // OfflineAudioContext, which renders as fast as the CPU allows instead of + // waiting for 1× real-time playback. +} diff --git a/src/lib/exporter/offlineAudioProcessor.ts b/src/lib/exporter/offlineAudioProcessor.ts new file mode 100644 index 00000000..7526db08 --- /dev/null +++ b/src/lib/exporter/offlineAudioProcessor.ts @@ -0,0 +1,444 @@ +import { SOURCE_AUDIO_NORMALIZE_GAIN } from "@/components/video-editor/audio/audioTypes"; +import type { + AudioRegion, + ClipRegion, + SourceAudioTrackSettings, + SpeedRegion, +} from "@/components/video-editor/types"; +import { buildResolvedAudioPlan } from "@/lib/exporter/audioRoutingEngine"; +import { estimateCompanionAudioStartDelaySeconds } from "@/lib/mediaTiming"; +import { AudioMediaProcessor } from "./audioMediaProcessor"; +import { + AUDIO_BITRATE, + ENCODE_BACKPRESSURE_LIMIT, + getSourceTrackIdFromPath, + MP4_AUDIO_CODEC, + OFFLINE_AUDIO_SAMPLE_RATE, + OFFLINE_CHUNK_DURATION_SEC, + OFFLINE_ENCODE_CHUNK_FRAMES, + type PreparedOfflineRender, + resolveSourceTrackGain, + softLimitOfflineMixPeaksInPlace, + type TimelineSlice, + type TrimLikeRegion, +} from "./audioProcessorShared"; +import type { VideoMuxer } from "./muxer"; + +export class OfflineAudioProcessor extends AudioMediaProcessor { + protected async renderAndMuxOfflineAudio( + videoUrl: string, + trimRegions: TrimLikeRegion[], + speedRegions: SpeedRegion[], + audioRegions: AudioRegion[], + sourceAudioFallbackPaths: string[], + sourceAudioFallbackStartDelayMsByPath: Record | undefined, + sourceAudioTrackSettings: SourceAudioTrackSettings | undefined, + clipRegions: ClipRegion[] | undefined, + muxer: VideoMuxer, + ): Promise { + const prepared = await this.prepareOfflineRender( + videoUrl, + trimRegions, + speedRegions, + audioRegions, + sourceAudioFallbackPaths, + sourceAudioFallbackStartDelayMsByPath, + sourceAudioTrackSettings, + clipRegions, + ); + if (this.cancelled) return; + await this.renderAndEncodeChunked(prepared, muxer); + } + + protected async prepareOfflineRender( + videoUrl: string, + trimRegions: TrimLikeRegion[], + speedRegions: SpeedRegion[], + audioRegions: AudioRegion[], + sourceAudioFallbackPaths: string[], + sourceAudioFallbackStartDelayMsByPath?: Record, + sourceAudioTrackSettings?: SourceAudioTrackSettings, + clipRegions?: ClipRegion[], + ): Promise { + if (this.cancelled) throw new Error("Export cancelled"); + this.onProgress?.(0); + + const resolvedPlan = buildResolvedAudioPlan({ + videoResource: videoUrl, + sourceAudioFallbackPaths, + audioRegions, + sourceTrackGainById: { + mic: resolveSourceTrackGain(sourceAudioTrackSettings, "mic"), + system: resolveSourceTrackGain(sourceAudioTrackSettings, "system"), + mixed: resolveSourceTrackGain(sourceAudioTrackSettings, "mixed"), + }, + embeddedGain: Math.max( + 0, + Math.min( + 2, + sourceAudioTrackSettings?.mixed + ? resolveSourceTrackGain(sourceAudioTrackSettings, "mixed") + : sourceAudioTrackSettings?.system + ? resolveSourceTrackGain(sourceAudioTrackSettings, "system") + : 1, + ), + ), + }); + + // Decode embedded source audio separately from companion sidecars. + const mainBuffer = resolvedPlan.includeEmbeddedInExport + ? await this.decodeAudioFromUrl(videoUrl) + : null; + const mainBufferGain = resolveSourceTrackGain(sourceAudioTrackSettings, "mixed"); + const mainBufferEntry = mainBuffer ? { buffer: mainBuffer, gain: mainBufferGain } : null; + if (this.cancelled) throw new Error("Export cancelled"); + + // Decode companion / sidecar audio files + const companionEntries: Array<{ + buffer: AudioBuffer; + startDelaySec: number; + gain: number; + }> = []; + const refDuration = + mainBuffer?.duration ?? + (resolvedPlan.playbackPaths.length > 0 ? await this.getMediaDurationSec(videoUrl) : 0); + for (const audioPath of resolvedPlan.playbackPaths) { + if (this.cancelled) throw new Error("Export cancelled"); + const buffer = await this.decodeAudioFromUrl(audioPath); + if (!buffer) continue; + + companionEntries.push({ + buffer, + gain: resolveSourceTrackGain( + sourceAudioTrackSettings, + getSourceTrackIdFromPath(audioPath), + ), + startDelaySec: estimateCompanionAudioStartDelaySeconds( + refDuration, + buffer.duration, + sourceAudioFallbackStartDelayMsByPath?.[audioPath], + ), + }); + } + if (this.cancelled) throw new Error("Export cancelled"); + + // Decode audio region overlay files + const regionEntries: Array<{ buffer: AudioBuffer; region: AudioRegion }> = []; + for (const region of audioRegions) { + if (this.cancelled) throw new Error("Export cancelled"); + const buffer = await this.decodeAudioFromUrl(region.audioPath); + if (buffer) regionEntries.push({ buffer, region }); + } + + this.onProgress?.(0.2); + + // Determine source duration for timeline calculation + const primaryBuffer = mainBufferEntry?.buffer ?? companionEntries[0]?.buffer ?? null; + if (!primaryBuffer && regionEntries.length === 0) { + throw new Error("No decodable audio sources found"); + } + + let sourceDurationSec: number; + if (mainBufferEntry?.buffer) { + sourceDurationSec = mainBufferEntry.buffer.duration; + } else if (resolvedPlan.playbackPaths.length > 0 || regionEntries.length > 0) { + sourceDurationSec = await this.getMediaDurationSec(videoUrl); + } else { + sourceDurationSec = primaryBuffer?.duration ?? 0; + } + const sourceDurationMs = sourceDurationSec * 1000; + + // Build timeline slices (non-trimmed segments with speed info) + const slices = this.buildTimelineSlices(sourceDurationMs, trimRegions, speedRegions); + + let outputDurationMs = 0; + for (const slice of slices) { + outputDurationMs += (slice.sourceEndMs - slice.sourceStartMs) / slice.speed; + } + + // Extend for audio regions that might exceed the video timeline + for (const { region } of regionEntries) { + const regionEndOutput = this.sourceTimeToOutputTime(region.endMs, slices); + outputDurationMs = Math.max(outputDurationMs, regionEndOutput); + } + + const numChannels = Math.min(primaryBuffer?.numberOfChannels ?? 2, 2); + const mutedSourceOutputRangesSec = (clipRegions ?? []) + .filter( + (clip) => + Boolean(clip.muted) && + Number.isFinite(clip.startMs) && + Number.isFinite(clip.endMs) && + clip.endMs > clip.startMs, + ) + .map((clip) => ({ + startSec: Math.max(0, clip.startMs / 1000), + endSec: Math.max(0, clip.endMs / 1000), + })); + + return { + mainBufferEntry, + companionEntries, + regionEntries, + mutedSourceOutputRangesSec, + slices, + outputDurationMs, + numChannels, + }; + } + + // Render timeline in chunks and encode each chunk to the muxer immediately. + // Memory is bounded to ~OFFLINE_CHUNK_DURATION_SEC of PCM per chunk + // instead of holding the entire output buffer in memory. + protected async renderAndEncodeChunked( + prepared: PreparedOfflineRender, + muxer: VideoMuxer, + ): Promise { + const { numChannels } = prepared; + const totalOutputSec = Math.max(prepared.outputDurationMs / 1000, 0.01); + + let encodeError: Error | null = null; + let muxError: Error | null = null; + let pendingMuxing = Promise.resolve(); + let wroteFirstChunk = false; + + const encodeConfig: AudioEncoderConfig = { + codec: MP4_AUDIO_CODEC, + sampleRate: OFFLINE_AUDIO_SAMPLE_RATE, + numberOfChannels: numChannels, + bitrate: AUDIO_BITRATE, + }; + + const supported = await AudioEncoder.isConfigSupported(encodeConfig); + if (!supported.supported) { + console.warn("[AudioProcessor] AAC encoding not supported for offline audio"); + return; + } + + const encoder = new AudioEncoder({ + output: (chunk: EncodedAudioChunk, meta?: EncodedAudioChunkMetadata) => { + pendingMuxing = pendingMuxing + .then(async () => { + if (this.cancelled) return; + await muxer.addAudioChunk(chunk, !wroteFirstChunk ? meta : undefined); + wroteFirstChunk = true; + }) + .catch((error) => { + muxError = error instanceof Error ? error : new Error(String(error)); + }); + }, + error: (error: DOMException) => { + encodeError = new Error(`Audio encode error: ${error.message}`); + }, + }); + encoder.configure(encodeConfig); + + try { + await this.renderChunked( + prepared, + totalOutputSec, + async (rendered, outputOffsetSec) => { + if (encodeError) throw encodeError; + if (muxError) throw muxError; + await this.feedBufferToEncoder(encoder, rendered, outputOffsetSec); + }, + ); + + if (encodeError) throw encodeError; + if (muxError) throw muxError; + + if (encoder.state === "configured") { + await encoder.flush(); + } + + await pendingMuxing; + + if (encodeError) throw encodeError; + if (muxError) throw muxError; + } finally { + if (encoder.state === "configured") { + encoder.close(); + } + } + } + + // Render timeline to a WAV blob for the native/FFmpeg export path. + // Processes in chunks to avoid holding the entire output in memory. + protected async renderToWavBlobChunked(prepared: PreparedOfflineRender): Promise { + const totalOutputSec = Math.max(prepared.outputDurationMs / 1000, 0.01); + const totalFrames = Math.ceil(totalOutputSec * OFFLINE_AUDIO_SAMPLE_RATE); + const numChannels = prepared.numChannels; + + const header = this.createWavHeader(OFFLINE_AUDIO_SAMPLE_RATE, numChannels, totalFrames); + const pcmParts: ArrayBuffer[] = [header]; + + await this.renderChunked(prepared, totalOutputSec, async (rendered) => { + pcmParts.push(...this.audioBufferToPcmParts(rendered)); + }); + + return new Blob(pcmParts, { type: "audio/wav" }); + } + + // Shared chunked rendering loop. Processes the timeline in + // OFFLINE_CHUNK_DURATION_SEC segments, calling onChunk for each rendered buffer. + protected async renderChunked( + prepared: PreparedOfflineRender, + totalOutputSec: number, + onChunk: ( + rendered: AudioBuffer, + outputOffsetSec: number, + chunkIndex: number, + ) => Promise, + ): Promise { + const { slices, numChannels } = prepared; + let outputOffsetSec = 0; + const chunkCount = Math.ceil(totalOutputSec / OFFLINE_CHUNK_DURATION_SEC); + + for (let i = 0; i < chunkCount && !this.cancelled; i++) { + const chunkSec = Math.min(OFFLINE_CHUNK_DURATION_SEC, totalOutputSec - outputOffsetSec); + const chunkFrames = Math.ceil(chunkSec * OFFLINE_AUDIO_SAMPLE_RATE); + + const offlineCtx = new OfflineAudioContext( + numChannels, + chunkFrames, + OFFLINE_AUDIO_SAMPLE_RATE, + ); + + // Schedule main audio + if (prepared.mainBufferEntry) { + this.scheduleBufferThroughTimeline( + offlineCtx, + prepared.mainBufferEntry.buffer, + slices, + 0, + prepared.mainBufferEntry.gain, + outputOffsetSec, + chunkSec, + prepared.mutedSourceOutputRangesSec, + ); + } + + // Schedule companion/sidecar audio + for (const entry of prepared.companionEntries) { + this.scheduleBufferThroughTimeline( + offlineCtx, + entry.buffer, + slices, + entry.startDelaySec, + entry.gain, + outputOffsetSec, + chunkSec, + prepared.mutedSourceOutputRangesSec, + ); + } + + // Schedule audio region overlays + for (const { buffer, region } of prepared.regionEntries) { + this.scheduleRegionForChunk( + offlineCtx, + buffer, + region, + slices, + outputOffsetSec, + chunkSec, + ); + } + + const rendered = await offlineCtx.startRendering(); + if (this.cancelled) break; + softLimitOfflineMixPeaksInPlace(rendered); + + await onChunk(rendered, outputOffsetSec, i); + + outputOffsetSec += chunkSec; + this.onProgress?.(0.3 + (outputOffsetSec / totalOutputSec) * 0.7); + } + } + + // Schedule an audio region overlay clipped to a specific chunk window. + protected scheduleRegionForChunk( + ctx: OfflineAudioContext, + buffer: AudioBuffer, + region: AudioRegion, + slices: TimelineSlice[], + chunkOutputStartSec: number, + chunkDurationSec: number, + ): void { + const outputStartMs = this.sourceTimeToOutputTime(region.startMs, slices); + const outputEndMs = this.sourceTimeToOutputTime(region.endMs, slices); + + let localStartSec = outputStartMs / 1000 - chunkOutputStartSec; + let localEndSec = outputEndMs / 1000 - chunkOutputStartSec; + + // Skip if region doesn't overlap with this chunk + if (localEndSec <= 0 || localStartSec >= chunkDurationSec) return; + + // Clip to chunk bounds + let bufferOffsetSec = 0; + if (localStartSec < 0) { + bufferOffsetSec = -localStartSec; + localStartSec = 0; + } + if (localEndSec > chunkDurationSec) { + localEndSec = chunkDurationSec; + } + + const duration = Math.min(localEndSec - localStartSec, buffer.duration - bufferOffsetSec); + if (duration <= 0.001) return; + + const gainNode = ctx.createGain(); + const normalizeGain = region.normalize ? SOURCE_AUDIO_NORMALIZE_GAIN : 1; + gainNode.gain.value = Math.max(0, Math.min(1, region.volume * normalizeGain)); + gainNode.connect(ctx.destination); + + const source = ctx.createBufferSource(); + source.buffer = buffer; + source.connect(gainNode); + source.start(localStartSec, bufferOffsetSec, duration); + } + + // Feed a rendered AudioBuffer chunk to an AudioEncoder with a timestamp offset. + protected async feedBufferToEncoder( + encoder: AudioEncoder, + buffer: AudioBuffer, + timestampOffsetSec: number, + ): Promise { + const sampleRate = buffer.sampleRate; + const numChannels = buffer.numberOfChannels; + const totalFrames = buffer.length; + + for ( + let offset = 0; + offset < totalFrames && !this.cancelled; + offset += OFFLINE_ENCODE_CHUNK_FRAMES + ) { + const frameCount = Math.min(OFFLINE_ENCODE_CHUNK_FRAMES, totalFrames - offset); + + const planarData = new Float32Array(frameCount * numChannels); + for (let ch = 0; ch < numChannels; ch++) { + const channelData = buffer.getChannelData(ch); + planarData.set(channelData.subarray(offset, offset + frameCount), ch * frameCount); + } + + const audioData = new AudioData({ + format: "f32-planar", + sampleRate, + numberOfFrames: frameCount, + numberOfChannels: numChannels, + timestamp: Math.round((offset / sampleRate + timestampOffsetSec) * 1_000_000), + data: planarData, + }); + + encoder.encode(audioData); + audioData.close(); + + while (encoder.encodeQueueSize >= ENCODE_BACKPRESSURE_LIMIT && !this.cancelled) { + await new Promise((r) => setTimeout(r, 1)); + } + } + } + + // Decode audio from a URL using streaming WebCodecs decode with bulk fallback. + // Streaming decode avoids holding the full compressed file in memory alongside + // the decoded AudioBuffer, reducing peak memory for large recordings. +} diff --git a/src/lib/exporter/streamingDecodePipeline.ts b/src/lib/exporter/streamingDecodePipeline.ts new file mode 100644 index 00000000..289bcf38 --- /dev/null +++ b/src/lib/exporter/streamingDecodePipeline.ts @@ -0,0 +1,448 @@ +import type { SpeedRegion, TrimRegion } from "@/components/video-editor/types"; +import { getEffectiveVideoStreamDurationSeconds } from "@/lib/mediaTiming"; +import type { WebDemuxer } from "web-demuxer"; +import { + buildVideoDecodeFailure, + getDecodedFrameTimelineOffsetUs, + preserveFirstVideoDecodeFailure, + type DecodedVideoInfo, + type VideoDecodeFailureContext, +} from "./streamingDecoderSupport"; +import { computeVideoSegments, splitVideoSegmentsBySpeed } from "./videoTimelineSegments"; + +type OnFrameCallback = ( + frame: VideoFrame, + exportTimestampUs: number, + sourceTimestampMs: number, + cursorTimestampMs: number, +) => Promise; + +interface StreamingDecodeContext { + demuxer: WebDemuxer; + decoder: VideoDecoder | null; + readonly cancelled: boolean; + readonly metadata: DecodedVideoInfo; + pendingFrames: VideoFrame[]; + readonly maxDecodeQueue: number; + readonly maxPendingFrames: number; +} + +const STARTUP_STABILIZATION_SECONDS = 1.25; +const STARTUP_MAX_DECODE_QUEUE = 12; +const STARTUP_MAX_PENDING_FRAMES = 28; + +export async function decodeVideoStream( + context: StreamingDecodeContext, + + targetFrameRate: number, + trimRegions: TrimRegion[] | undefined, + speedRegions: SpeedRegion[] | undefined, + onFrame: OnFrameCallback, +): Promise { + if (!context.demuxer || !context.metadata) { + throw new Error("Must call loadMetadata() before decodeAll()"); + } + + const decoderConfig = await context.demuxer.getDecoderConfig("video"); + const codec = context.metadata.codec.toLowerCase(); + const shouldPreferSoftwareDecode = codec.includes("av01") || codec.includes("av1"); + const effectiveVideoDuration = getEffectiveVideoStreamDurationSeconds({ + duration: context.metadata.duration, + streamDuration: context.metadata.streamDuration, + }); + const segments = splitVideoSegmentsBySpeed( + computeVideoSegments(effectiveVideoDuration, trimRegions), + speedRegions, + ); + const segmentOutputFrameCounts = segments.map((segment) => + Math.ceil(((segment.endSec - segment.startSec) / segment.speed) * targetFrameRate), + ); + const expectedOutputFrames = segmentOutputFrameCounts.reduce((sum, count) => sum + count, 0); + const frameDurationUs = 1_000_000 / targetFrameRate; + const epsilonSec = 0.001; + const startupStabilizationSeconds = STARTUP_STABILIZATION_SECONDS; + const startupFrameBudget = Math.max( + 1, + Math.round(targetFrameRate * startupStabilizationSeconds), + ); + let exportFrameIndex = 0; + let loggedSteadyStateBackpressure = false; + const backpressureWaiters = new Set<() => void>(); + + const notifyBackpressureProgress = () => { + if (backpressureWaiters.size === 0) { + return; + } + + const waiters = [...backpressureWaiters]; + backpressureWaiters.clear(); + for (const resolve of waiters) { + resolve(); + } + }; + + const waitForBackpressureProgress = () => + new Promise((resolve) => { + backpressureWaiters.add(resolve); + }); + + console.log( + `[StreamingVideoDecoder] Startup-safe decode backpressure active for first ${startupStabilizationSeconds}s (${startupFrameBudget} frames)`, + ); + + // Async frame queue — decoder pushes, consumer pulls + context.pendingFrames.length = 0; + const pendingFrames = context.pendingFrames; + let frameResolve: ((frame: VideoFrame | null) => void) | null = null; + let decodeError: Error | null = null; + let decodeDone = false; + let firstDecodedFrameTimestampUs: number | null = null; + let decodedFrameTimelineOffsetUs = 0; + let submittedChunkCount = 0; + let lastSubmittedChunk: EncodedVideoChunk | undefined; + let lastSubmittedChunkIndex: number | undefined; + const preferredDecoderConfig = shouldPreferSoftwareDecode + ? { + ...decoderConfig, + hardwareAcceleration: "prefer-software" as const, + } + : decoderConfig; + let activeDecoderConfig = preferredDecoderConfig; + const getDecoderFailureContext = (): VideoDecodeFailureContext => ({ + decoderConfig: activeDecoderConfig, + sourceMetadata: context.metadata ?? undefined, + chunkIndex: lastSubmittedChunkIndex, + chunk: lastSubmittedChunk, + decoderState: context.decoder?.state, + decodeQueueSize: context.decoder?.decodeQueueSize, + }); + const recordFirstDecodeError = (error: unknown) => { + decodeError = preserveFirstVideoDecodeFailure( + decodeError, + error, + getDecoderFailureContext(), + ); + }; + + context.decoder = new VideoDecoder({ + output: (frame: VideoFrame) => { + if (frameResolve) { + const resolve = frameResolve; + frameResolve = null; + resolve(frame); + } else { + pendingFrames.push(frame); + } + notifyBackpressureProgress(); + }, + error: (e: DOMException) => { + recordFirstDecodeError(e); + if (frameResolve) { + const resolve = frameResolve; + frameResolve = null; + resolve(null); + } + notifyBackpressureProgress(); + }, + }); + try { + context.decoder.configure(preferredDecoderConfig); + } catch (error) { + if (!shouldPreferSoftwareDecode) { + throw buildVideoDecodeFailure(error, getDecoderFailureContext()); + } + // Fall back to default decoder config if software preference is unsupported. + activeDecoderConfig = decoderConfig; + try { + context.decoder.configure(decoderConfig); + } catch (fallbackError) { + throw buildVideoDecodeFailure(fallbackError, getDecoderFailureContext()); + } + } + + const getNextFrame = (): Promise => { + if (decodeError) return Promise.resolve(null); + if (pendingFrames.length > 0) { + const frame = pendingFrames.shift()!; + notifyBackpressureProgress(); + return Promise.resolve(frame); + } + if (decodeDone) return Promise.resolve(null); + return new Promise((resolve) => { + frameResolve = resolve; + }); + }; + + // One forward stream through the whole file. + // Pass explicit range because some containers are truncated when no end is provided. + const readEndSec = + Math.max( + context.metadata.duration + (context.metadata.mediaStartTime ?? 0), + (context.metadata.streamDuration ?? context.metadata.duration) + + (context.metadata.streamStartTime ?? context.metadata.mediaStartTime ?? 0), + ) + 0.5; + const reader = context.demuxer.read("video", 0, readEndSec).getReader(); + + // Feed chunks to decoder in background with backpressure + const feedPromise = (async () => { + try { + while (!context.cancelled && !decodeError) { + const { done, value: chunk } = await reader.read(); + if (done || !chunk) break; + + if (!loggedSteadyStateBackpressure && exportFrameIndex >= startupFrameBudget) { + loggedSteadyStateBackpressure = true; + console.log( + "[StreamingVideoDecoder] Switched to steady-state decode backpressure", + ); + } + + const decodeQueueLimit = + exportFrameIndex < startupFrameBudget + ? Math.min(context.maxDecodeQueue, STARTUP_MAX_DECODE_QUEUE) + : context.maxDecodeQueue; + const pendingFrameLimit = + exportFrameIndex < startupFrameBudget + ? Math.min(context.maxPendingFrames, STARTUP_MAX_PENDING_FRAMES) + : context.maxPendingFrames; + + // Backpressure on both decode queue and decoded frame backlog. + while ( + !decodeError && + context.decoder!.state === "configured" && + (context.decoder!.decodeQueueSize > decodeQueueLimit || + pendingFrames.length > pendingFrameLimit) && + !context.cancelled + ) { + await waitForBackpressureProgress(); + } + if (context.cancelled || decodeError) break; + if (context.decoder!.state !== "configured") { + recordFirstDecodeError( + new DOMException( + "Decoder closed before the next video chunk was submitted.", + "InvalidStateError", + ), + ); + break; + } + + lastSubmittedChunk = chunk; + lastSubmittedChunkIndex = submittedChunkCount; + context.decoder!.decode(chunk); + submittedChunkCount++; + } + + if (!context.cancelled && context.decoder!.state === "configured") { + await context.decoder!.flush(); + } + } catch (e) { + recordFirstDecodeError(e); + } finally { + decodeDone = true; + if (frameResolve) { + const resolve = frameResolve; + frameResolve = null; + resolve(null); + } + notifyBackpressureProgress(); + } + })(); + + // Route decoded frames into segments by timestamp, then deliver with VFR→CFR resampling + let segmentIdx = 0; + let segmentFrameIndex = 0; + let lastDecodedFrameSec: number | null = null; + let heldFrame: VideoFrame | null = null; + let heldFrameSec = 0; + + const emitHeldFrameForTarget = async (segment: { + startSec: number; + endSec: number; + speed: number; + }) => { + if (!heldFrame) return false; + const segmentFrameCount = segmentOutputFrameCounts[segmentIdx]; + if (segmentFrameIndex >= segmentFrameCount) return false; + + const segmentDurationSec = segment.endSec - segment.startSec; + const sourceTimeSec = + segment.startSec + (segmentFrameIndex / segmentFrameCount) * segmentDurationSec; + if (sourceTimeSec >= segment.endSec - epsilonSec) return false; + + const sourceTimestampMs = sourceTimeSec * 1000; + await onFrame( + heldFrame, + exportFrameIndex * frameDurationUs, + sourceTimestampMs, + sourceTimestampMs, + ); + segmentFrameIndex++; + exportFrameIndex++; + return true; + }; + + while (!context.cancelled && segmentIdx < segments.length) { + const frame = await getNextFrame(); + if (!frame) break; + + if (firstDecodedFrameTimestampUs === null) { + firstDecodedFrameTimestampUs = frame.timestamp; + decodedFrameTimelineOffsetUs = getDecodedFrameTimelineOffsetUs( + firstDecodedFrameTimestampUs, + context.metadata, + ); + } + + const normalizedFrameTimeSec = Math.max( + 0, + (frame.timestamp - firstDecodedFrameTimestampUs + decodedFrameTimelineOffsetUs) / + 1_000_000, + ); + const frameTimeSec: number = + lastDecodedFrameSec === null + ? normalizedFrameTimeSec + : Math.max(lastDecodedFrameSec, normalizedFrameTimeSec); + lastDecodedFrameSec = frameTimeSec; + + // Finalize completed segments before handling this frame. + while ( + segmentIdx < segments.length && + frameTimeSec >= segments[segmentIdx].endSec - epsilonSec + ) { + const segment = segments[segmentIdx]; + while (!context.cancelled && (await emitHeldFrameForTarget(segment))) { + // Keep emitting remaining output frames for this segment from the last known frame. + } + + segmentIdx++; + segmentFrameIndex = 0; + if ( + heldFrame && + segmentIdx < segments.length && + heldFrameSec < segments[segmentIdx].startSec - epsilonSec + ) { + heldFrame.close(); + heldFrame = null; + } + } + + if (segmentIdx >= segments.length) { + frame.close(); + continue; + } + + const currentSegment = segments[segmentIdx]; + + // Before current segment (trimmed region or pre-roll). + if (frameTimeSec < currentSegment.startSec - epsilonSec) { + frame.close(); + continue; + } + + if (!heldFrame) { + heldFrame = frame; + heldFrameSec = frameTimeSec; + continue; + } + + // Any target timestamp before this midpoint is closer to heldFrame than current frame. + const handoffBoundarySec = (heldFrameSec + frameTimeSec) / 2; + while (!context.cancelled) { + const segmentFrameCount = segmentOutputFrameCounts[segmentIdx]; + if (segmentFrameIndex >= segmentFrameCount) { + break; + } + + const segmentDurationSec = currentSegment.endSec - currentSegment.startSec; + const sourceTimeSec = + currentSegment.startSec + + (segmentFrameIndex / segmentFrameCount) * segmentDurationSec; + if (sourceTimeSec >= currentSegment.endSec - epsilonSec) { + break; + } + if (sourceTimeSec > handoffBoundarySec) { + break; + } + + const sourceTimestampMs = sourceTimeSec * 1000; + await onFrame( + heldFrame, + exportFrameIndex * frameDurationUs, + sourceTimestampMs, + sourceTimestampMs, + ); + segmentFrameIndex++; + exportFrameIndex++; + } + + heldFrame.close(); + heldFrame = frame; + heldFrameSec = frameTimeSec; + } + + // Flush remaining output frames for the last decoded frame. + if (!decodeError && heldFrame && segmentIdx < segments.length) { + while (!context.cancelled && segmentIdx < segments.length) { + const segment = segments[segmentIdx]; + if (heldFrameSec < segment.startSec - epsilonSec) { + break; + } + + while (!context.cancelled && (await emitHeldFrameForTarget(segment))) { + // Keep emitting output frames for the active segment. + } + + segmentIdx++; + segmentFrameIndex = 0; + if ( + segmentIdx < segments.length && + heldFrameSec < segments[segmentIdx].startSec - epsilonSec + ) { + break; + } + } + heldFrame.close(); + heldFrame = null; + } else if (heldFrame) { + heldFrame.close(); + heldFrame = null; + } + + // Drain leftover decoded frames + while (!decodeDone && !decodeError) { + const frame = await getNextFrame(); + if (!frame) break; + frame.close(); + } + + try { + reader.cancel(); + } catch { + /* already closed */ + } + await feedPromise; + for (const f of pendingFrames) f.close(); + pendingFrames.length = 0; + + if (context.decoder?.state === "configured") { + context.decoder.close(); + } + context.decoder = null; + + if (decodeError) { + throw decodeError; + } + + const requiredEndSec = segments.length > 0 ? segments[segments.length - 1].endSec : 0; + if ( + !context.cancelled && + lastDecodedFrameSec !== null && + requiredEndSec - lastDecodedFrameSec > 1 && + exportFrameIndex < expectedOutputFrames + ) { + throw new Error( + `Video decode ended early at ${lastDecodedFrameSec.toFixed(3)}s (needed ${requiredEndSec.toFixed(3)}s; rendered ${exportFrameIndex}/${expectedOutputFrames} frames).`, + ); + } +} diff --git a/src/lib/exporter/streamingDecoder.ts b/src/lib/exporter/streamingDecoder.ts index ee59a0ac..152fdb19 100644 --- a/src/lib/exporter/streamingDecoder.ts +++ b/src/lib/exporter/streamingDecoder.ts @@ -2,120 +2,27 @@ import { WebDemuxer } from "web-demuxer"; import type { SpeedRegion, TrimRegion } from "@/components/video-editor/types"; import { getEffectiveVideoStreamDurationSeconds } from "@/lib/mediaTiming"; import { createFallbackDemuxerSource, resolveMediaResourceUrl } from "./localMediaSource"; +import { decodeVideoStream } from "./streamingDecodePipeline"; +import { computeVideoSegments, splitVideoSegmentsBySpeed } from "./videoTimelineSegments"; const DEFAULT_MAX_DECODE_QUEUE = 12; const DEFAULT_MAX_PENDING_FRAMES = 32; -const STARTUP_STABILIZATION_SECONDS = 1.25; -const STARTUP_MAX_DECODE_QUEUE = 12; -const STARTUP_MAX_PENDING_FRAMES = 28; -export interface DecodedVideoInfo { - width: number; - height: number; - duration: number; // seconds - mediaStartTime?: number; // seconds - streamStartTime?: number; // seconds - streamDuration?: number; // seconds - frameRate: number; - codec: string; - hasAudio: boolean; - audioCodec?: string; - audioSampleRate?: number; -} +import type { DecodedVideoInfo } from "./streamingDecoderSupport"; +export { + buildVideoDecodeFailure, + getDecodedFrameStartupOffsetUs, + getDecodedFrameTimelineOffsetUs, + getVideoDecodeFailureCode, + preserveFirstVideoDecodeFailure, + type DecodedVideoInfo, + type VideoDecodeFailureContext, +} from "./streamingDecoderSupport"; interface StreamingVideoDecoderLoadOptions { useFallbackMediaSource?: boolean; } -interface VideoDecodeFailureContext { - decoderConfig: VideoDecoderConfig; - sourceMetadata?: DecodedVideoInfo; - chunkIndex?: number; - chunk?: EncodedVideoChunk; - decoderState?: CodecState; - decodeQueueSize?: number; -} - -/** Maps WebCodecs failures to stable support-facing identifiers. */ -export function getVideoDecodeFailureCode(error: unknown): string { - const name = error instanceof DOMException ? error.name : ""; - switch (name) { - case "EncodingError": - return "VIDEO_DECODE_ENCODING_ERROR"; - case "NotSupportedError": - return "VIDEO_CODEC_UNSUPPORTED"; - case "QuotaExceededError": - return "VIDEO_DECODER_RESOURCE_EXHAUSTED"; - case "InvalidStateError": - return "VIDEO_DECODER_INVALID_STATE"; - default: - return "VIDEO_DECODE_FAILED"; - } -} - -function describeUnknownError(error: unknown): string { - if (error instanceof DOMException) { - return `${error.name}: ${error.message}`; - } - - if (error instanceof Error) { - return error.message; - } - - return String(error); -} - -/** Builds a decode error with codec, source, chunk, and decoder-state context. */ -export function buildVideoDecodeFailure(error: unknown, context: VideoDecodeFailureContext): Error { - const details = [`codec=${context.decoderConfig.codec}`]; - const failureCode = getVideoDecodeFailureCode(error); - const width = context.decoderConfig.codedWidth; - const height = context.decoderConfig.codedHeight; - if (width && height) { - details.push(`codedSize=${width}x${height}`); - } - if (context.decoderConfig.hardwareAcceleration) { - details.push(`hardwareAcceleration=${context.decoderConfig.hardwareAcceleration}`); - } - if (context.sourceMetadata) { - details.push(`sourceFps=${context.sourceMetadata.frameRate}`); - details.push(`sourceDurationSec=${context.sourceMetadata.duration}`); - } - if (context.chunkIndex !== undefined) { - details.push(`chunkIndex=${context.chunkIndex}`); - } - if (context.chunk) { - details.push(`chunkType=${context.chunk.type}`); - details.push(`chunkTimestampUs=${context.chunk.timestamp}`); - details.push(`sourceTimeSec=${(context.chunk.timestamp / 1_000_000).toFixed(3)}`); - if (typeof context.chunk.duration === "number") { - details.push(`chunkDurationUs=${context.chunk.duration}`); - } - details.push(`chunkBytes=${context.chunk.byteLength}`); - } - if (context.decoderState) { - details.push(`decoderState=${context.decoderState}`); - } - if (context.decodeQueueSize !== undefined) { - details.push(`decodeQueueSize=${context.decodeQueueSize}`); - } - - const failure = new Error( - `[${failureCode}] VideoDecoder failure: ${describeUnknownError(error)} (${details.join(", ")})`, - ); - (failure as Error & { cause?: unknown }).cause = error; - return failure; -} - -/** Keeps the original decoder failure when cleanup triggers secondary errors. */ -export function preserveFirstVideoDecodeFailure( - existingError: Error | null, - error: unknown, - context: VideoDecodeFailureContext, -): Error { - return existingError ?? buildVideoDecodeFailure(error, context); -} - /** Decoder retains ownership of the VideoFrame and closes it after use. */ type OnFrameCallback = ( frame: VideoFrame, @@ -124,32 +31,6 @@ type OnFrameCallback = ( cursorTimestampMs: number, ) => Promise; -export function getDecodedFrameStartupOffsetUs( - firstDecodedFrameTimestampUs: number, - metadata: Pick, -): number { - const streamStartTimeUs = Math.round( - (metadata.streamStartTime ?? metadata.mediaStartTime ?? 0) * 1_000_000, - ); - - return Math.max(0, firstDecodedFrameTimestampUs - streamStartTimeUs); -} - -export function getDecodedFrameTimelineOffsetUs( - firstDecodedFrameTimestampUs: number, - metadata: Pick, -): number { - const mediaStartTimeUs = Math.round((metadata.mediaStartTime ?? 0) * 1_000_000); - const streamStartTimeUs = Math.round( - (metadata.streamStartTime ?? metadata.mediaStartTime ?? 0) * 1_000_000, - ); - - return ( - Math.max(0, streamStartTimeUs - mediaStartTimeUs) + - getDecodedFrameStartupOffsetUs(firstDecodedFrameTimestampUs, metadata) - ); -} - /** * Decodes video frames via web-demuxer + VideoDecoder in a single forward pass. * Way faster than seeking an HTMLVideoElement per frame. @@ -291,498 +172,44 @@ export class StreamingVideoDecoder { throw new Error("Must call loadMetadata() before decodeAll()"); } - const decoderConfig = await this.demuxer.getDecoderConfig("video"); - const codec = this.metadata.codec.toLowerCase(); - const shouldPreferSoftwareDecode = codec.includes("av01") || codec.includes("av1"); - const effectiveVideoDuration = getEffectiveVideoStreamDurationSeconds({ - duration: this.metadata.duration, - streamDuration: this.metadata.streamDuration, - }); - const segments = this.splitBySpeed( - this.computeSegments(effectiveVideoDuration, trimRegions), + const owner = this; + await decodeVideoStream( + { + demuxer: this.demuxer, + metadata: this.metadata, + pendingFrames: this.pendingFrames, + maxDecodeQueue: this.maxDecodeQueue, + maxPendingFrames: this.maxPendingFrames, + get cancelled() { + return owner.cancelled; + }, + get decoder() { + return owner.decoder; + }, + set decoder(value) { + owner.decoder = value; + }, + }, + targetFrameRate, + trimRegions, speedRegions, + onFrame, ); - const segmentOutputFrameCounts = segments.map((segment) => - Math.ceil(((segment.endSec - segment.startSec) / segment.speed) * targetFrameRate), - ); - const expectedOutputFrames = segmentOutputFrameCounts.reduce( - (sum, count) => sum + count, - 0, - ); - const frameDurationUs = 1_000_000 / targetFrameRate; - const epsilonSec = 0.001; - const startupStabilizationSeconds = STARTUP_STABILIZATION_SECONDS; - const startupFrameBudget = Math.max( - 1, - Math.round(targetFrameRate * startupStabilizationSeconds), - ); - let exportFrameIndex = 0; - let loggedSteadyStateBackpressure = false; - const backpressureWaiters = new Set<() => void>(); - - const notifyBackpressureProgress = () => { - if (backpressureWaiters.size === 0) { - return; - } - - const waiters = [...backpressureWaiters]; - backpressureWaiters.clear(); - for (const resolve of waiters) { - resolve(); - } - }; - - const waitForBackpressureProgress = () => - new Promise((resolve) => { - backpressureWaiters.add(resolve); - }); - - console.log( - `[StreamingVideoDecoder] Startup-safe decode backpressure active for first ${startupStabilizationSeconds}s (${startupFrameBudget} frames)`, - ); - - // Async frame queue — decoder pushes, consumer pulls - this.pendingFrames.length = 0; - const pendingFrames = this.pendingFrames; - let frameResolve: ((frame: VideoFrame | null) => void) | null = null; - let decodeError: Error | null = null; - let decodeDone = false; - let firstDecodedFrameTimestampUs: number | null = null; - let decodedFrameTimelineOffsetUs = 0; - let submittedChunkCount = 0; - let lastSubmittedChunk: EncodedVideoChunk | undefined; - let lastSubmittedChunkIndex: number | undefined; - const preferredDecoderConfig = shouldPreferSoftwareDecode - ? { - ...decoderConfig, - hardwareAcceleration: "prefer-software" as const, - } - : decoderConfig; - let activeDecoderConfig = preferredDecoderConfig; - const getDecoderFailureContext = (): VideoDecodeFailureContext => ({ - decoderConfig: activeDecoderConfig, - sourceMetadata: this.metadata ?? undefined, - chunkIndex: lastSubmittedChunkIndex, - chunk: lastSubmittedChunk, - decoderState: this.decoder?.state, - decodeQueueSize: this.decoder?.decodeQueueSize, - }); - const recordFirstDecodeError = (error: unknown) => { - decodeError = preserveFirstVideoDecodeFailure( - decodeError, - error, - getDecoderFailureContext(), - ); - }; - - this.decoder = new VideoDecoder({ - output: (frame: VideoFrame) => { - if (frameResolve) { - const resolve = frameResolve; - frameResolve = null; - resolve(frame); - } else { - pendingFrames.push(frame); - } - notifyBackpressureProgress(); - }, - error: (e: DOMException) => { - recordFirstDecodeError(e); - if (frameResolve) { - const resolve = frameResolve; - frameResolve = null; - resolve(null); - } - notifyBackpressureProgress(); - }, - }); - try { - this.decoder.configure(preferredDecoderConfig); - } catch (error) { - if (!shouldPreferSoftwareDecode) { - throw buildVideoDecodeFailure(error, getDecoderFailureContext()); - } - // Fall back to default decoder config if software preference is unsupported. - activeDecoderConfig = decoderConfig; - try { - this.decoder.configure(decoderConfig); - } catch (fallbackError) { - throw buildVideoDecodeFailure(fallbackError, getDecoderFailureContext()); - } - } - - const getNextFrame = (): Promise => { - if (decodeError) return Promise.resolve(null); - if (pendingFrames.length > 0) { - const frame = pendingFrames.shift()!; - notifyBackpressureProgress(); - return Promise.resolve(frame); - } - if (decodeDone) return Promise.resolve(null); - return new Promise((resolve) => { - frameResolve = resolve; - }); - }; - - // One forward stream through the whole file. - // Pass explicit range because some containers are truncated when no end is provided. - const readEndSec = - Math.max( - this.metadata.duration + (this.metadata.mediaStartTime ?? 0), - (this.metadata.streamDuration ?? this.metadata.duration) + - (this.metadata.streamStartTime ?? this.metadata.mediaStartTime ?? 0), - ) + 0.5; - const reader = this.demuxer.read("video", 0, readEndSec).getReader(); - - // Feed chunks to decoder in background with backpressure - const feedPromise = (async () => { - try { - while (!this.cancelled && !decodeError) { - const { done, value: chunk } = await reader.read(); - if (done || !chunk) break; - - if (!loggedSteadyStateBackpressure && exportFrameIndex >= startupFrameBudget) { - loggedSteadyStateBackpressure = true; - console.log( - "[StreamingVideoDecoder] Switched to steady-state decode backpressure", - ); - } - - const decodeQueueLimit = - exportFrameIndex < startupFrameBudget - ? Math.min(this.maxDecodeQueue, STARTUP_MAX_DECODE_QUEUE) - : this.maxDecodeQueue; - const pendingFrameLimit = - exportFrameIndex < startupFrameBudget - ? Math.min(this.maxPendingFrames, STARTUP_MAX_PENDING_FRAMES) - : this.maxPendingFrames; - - // Backpressure on both decode queue and decoded frame backlog. - while ( - !decodeError && - this.decoder!.state === "configured" && - (this.decoder!.decodeQueueSize > decodeQueueLimit || - pendingFrames.length > pendingFrameLimit) && - !this.cancelled - ) { - await waitForBackpressureProgress(); - } - if (this.cancelled || decodeError) break; - if (this.decoder!.state !== "configured") { - recordFirstDecodeError( - new DOMException( - "Decoder closed before the next video chunk was submitted.", - "InvalidStateError", - ), - ); - break; - } - - lastSubmittedChunk = chunk; - lastSubmittedChunkIndex = submittedChunkCount; - this.decoder!.decode(chunk); - submittedChunkCount++; - } - - if (!this.cancelled && this.decoder!.state === "configured") { - await this.decoder!.flush(); - } - } catch (e) { - recordFirstDecodeError(e); - } finally { - decodeDone = true; - if (frameResolve) { - const resolve = frameResolve; - frameResolve = null; - resolve(null); - } - notifyBackpressureProgress(); - } - })(); - - // Route decoded frames into segments by timestamp, then deliver with VFR→CFR resampling - let segmentIdx = 0; - let segmentFrameIndex = 0; - let lastDecodedFrameSec: number | null = null; - let heldFrame: VideoFrame | null = null; - let heldFrameSec = 0; - - const emitHeldFrameForTarget = async (segment: { - startSec: number; - endSec: number; - speed: number; - }) => { - if (!heldFrame) return false; - const segmentFrameCount = segmentOutputFrameCounts[segmentIdx]; - if (segmentFrameIndex >= segmentFrameCount) return false; - - const segmentDurationSec = segment.endSec - segment.startSec; - const sourceTimeSec = - segment.startSec + (segmentFrameIndex / segmentFrameCount) * segmentDurationSec; - if (sourceTimeSec >= segment.endSec - epsilonSec) return false; - - const sourceTimestampMs = sourceTimeSec * 1000; - await onFrame( - heldFrame, - exportFrameIndex * frameDurationUs, - sourceTimestampMs, - sourceTimestampMs, - ); - segmentFrameIndex++; - exportFrameIndex++; - return true; - }; - - while (!this.cancelled && segmentIdx < segments.length) { - const frame = await getNextFrame(); - if (!frame) break; - - if (firstDecodedFrameTimestampUs === null) { - firstDecodedFrameTimestampUs = frame.timestamp; - decodedFrameTimelineOffsetUs = getDecodedFrameTimelineOffsetUs( - firstDecodedFrameTimestampUs, - this.metadata, - ); - } - - const normalizedFrameTimeSec = Math.max( - 0, - (frame.timestamp - firstDecodedFrameTimestampUs + decodedFrameTimelineOffsetUs) / - 1_000_000, - ); - const frameTimeSec: number = - lastDecodedFrameSec === null - ? normalizedFrameTimeSec - : Math.max(lastDecodedFrameSec, normalizedFrameTimeSec); - lastDecodedFrameSec = frameTimeSec; - - // Finalize completed segments before handling this frame. - while ( - segmentIdx < segments.length && - frameTimeSec >= segments[segmentIdx].endSec - epsilonSec - ) { - const segment = segments[segmentIdx]; - while (!this.cancelled && (await emitHeldFrameForTarget(segment))) { - // Keep emitting remaining output frames for this segment from the last known frame. - } - - segmentIdx++; - segmentFrameIndex = 0; - if ( - heldFrame && - segmentIdx < segments.length && - heldFrameSec < segments[segmentIdx].startSec - epsilonSec - ) { - heldFrame.close(); - heldFrame = null; - } - } - - if (segmentIdx >= segments.length) { - frame.close(); - continue; - } - - const currentSegment = segments[segmentIdx]; - - // Before current segment (trimmed region or pre-roll). - if (frameTimeSec < currentSegment.startSec - epsilonSec) { - frame.close(); - continue; - } - - if (!heldFrame) { - heldFrame = frame; - heldFrameSec = frameTimeSec; - continue; - } - - // Any target timestamp before this midpoint is closer to heldFrame than current frame. - const handoffBoundarySec = (heldFrameSec + frameTimeSec) / 2; - while (!this.cancelled) { - const segmentFrameCount = segmentOutputFrameCounts[segmentIdx]; - if (segmentFrameIndex >= segmentFrameCount) { - break; - } - - const segmentDurationSec = currentSegment.endSec - currentSegment.startSec; - const sourceTimeSec = - currentSegment.startSec + - (segmentFrameIndex / segmentFrameCount) * segmentDurationSec; - if (sourceTimeSec >= currentSegment.endSec - epsilonSec) { - break; - } - if (sourceTimeSec > handoffBoundarySec) { - break; - } - - const sourceTimestampMs = sourceTimeSec * 1000; - await onFrame( - heldFrame, - exportFrameIndex * frameDurationUs, - sourceTimestampMs, - sourceTimestampMs, - ); - segmentFrameIndex++; - exportFrameIndex++; - } - - heldFrame.close(); - heldFrame = frame; - heldFrameSec = frameTimeSec; - } - - // Flush remaining output frames for the last decoded frame. - if (!decodeError && heldFrame && segmentIdx < segments.length) { - while (!this.cancelled && segmentIdx < segments.length) { - const segment = segments[segmentIdx]; - if (heldFrameSec < segment.startSec - epsilonSec) { - break; - } - - while (!this.cancelled && (await emitHeldFrameForTarget(segment))) { - // Keep emitting output frames for the active segment. - } - - segmentIdx++; - segmentFrameIndex = 0; - if ( - segmentIdx < segments.length && - heldFrameSec < segments[segmentIdx].startSec - epsilonSec - ) { - break; - } - } - heldFrame.close(); - heldFrame = null; - } else if (heldFrame) { - heldFrame.close(); - heldFrame = null; - } - - // Drain leftover decoded frames - while (!decodeDone && !decodeError) { - const frame = await getNextFrame(); - if (!frame) break; - frame.close(); - } - - try { - reader.cancel(); - } catch { - /* already closed */ - } - await feedPromise; - for (const f of pendingFrames) f.close(); - pendingFrames.length = 0; - - if (this.decoder?.state === "configured") { - this.decoder.close(); - } - this.decoder = null; - - if (decodeError) { - throw decodeError; - } - - const requiredEndSec = segments.length > 0 ? segments[segments.length - 1].endSec : 0; - if ( - !this.cancelled && - lastDecodedFrameSec !== null && - requiredEndSec - lastDecodedFrameSec > 1 && - exportFrameIndex < expectedOutputFrames - ) { - throw new Error( - `Video decode ended early at ${lastDecodedFrameSec.toFixed(3)}s (needed ${requiredEndSec.toFixed(3)}s; rendered ${exportFrameIndex}/${expectedOutputFrames} frames).`, - ); - } - } - - private computeSegments( - totalDuration: number, - trimRegions?: TrimRegion[], - ): Array<{ startSec: number; endSec: number }> { - if (!trimRegions || trimRegions.length === 0) { - return [{ startSec: 0, endSec: totalDuration }]; - } - - const sorted = [...trimRegions].sort((a, b) => a.startMs - b.startMs); - const segments: Array<{ startSec: number; endSec: number }> = []; - let cursor = 0; - - for (const trim of sorted) { - const trimStart = trim.startMs / 1000; - const trimEnd = trim.endMs / 1000; - if (cursor < trimStart) { - segments.push({ startSec: cursor, endSec: trimStart }); - } - cursor = Math.max(cursor, trimEnd); - } - - if (cursor < totalDuration) { - segments.push({ startSec: cursor, endSec: totalDuration }); - } - - return segments; } getEffectiveDuration(trimRegions?: TrimRegion[], speedRegions?: SpeedRegion[]): number { if (!this.metadata) throw new Error("Must call loadMetadata() first"); - const trimSegments = this.computeSegments( + const trimSegments = computeVideoSegments( getEffectiveVideoStreamDurationSeconds({ duration: this.metadata.duration, streamDuration: this.metadata.streamDuration, }), trimRegions, ); - const speedSegments = this.splitBySpeed(trimSegments, speedRegions); + const speedSegments = splitVideoSegmentsBySpeed(trimSegments, speedRegions); return speedSegments.reduce((sum, seg) => sum + (seg.endSec - seg.startSec) / seg.speed, 0); } - private splitBySpeed( - segments: Array<{ startSec: number; endSec: number }>, - speedRegions?: SpeedRegion[], - ): Array<{ startSec: number; endSec: number; speed: number }> { - if (!speedRegions || speedRegions.length === 0) - return segments.map((s) => ({ ...s, speed: 1 })); - - const result: Array<{ startSec: number; endSec: number; speed: number }> = []; - for (const segment of segments) { - const overlapping = speedRegions - .filter( - (sr) => - sr.startMs / 1000 < segment.endSec && sr.endMs / 1000 > segment.startSec, - ) - .sort((a, b) => a.startMs - b.startMs); - - if (overlapping.length === 0) { - result.push({ ...segment, speed: 1 }); - continue; - } - - let cursor = segment.startSec; - for (const sr of overlapping) { - const srStart = Math.max(sr.startMs / 1000, segment.startSec); - const srEnd = Math.min(sr.endMs / 1000, segment.endSec); - if (cursor < srStart) { - result.push({ startSec: cursor, endSec: srStart, speed: 1 }); - } - const effectiveStart = Math.max(cursor, srStart); - if (srEnd > effectiveStart) { - result.push({ - startSec: effectiveStart, - endSec: srEnd, - speed: sr.speed, - }); - } - cursor = Math.max(cursor, srEnd); - } - if (cursor < segment.endSec) - result.push({ startSec: cursor, endSec: segment.endSec, speed: 1 }); - } - return result.filter((s) => s.endSec - s.startSec > 0.0001); - } - cancel(): void { this.cancelled = true; } diff --git a/src/lib/exporter/streamingDecoderSupport.ts b/src/lib/exporter/streamingDecoderSupport.ts new file mode 100644 index 00000000..dd934631 --- /dev/null +++ b/src/lib/exporter/streamingDecoderSupport.ts @@ -0,0 +1,128 @@ +export interface DecodedVideoInfo { + width: number; + height: number; + duration: number; // seconds + mediaStartTime?: number; // seconds + streamStartTime?: number; // seconds + streamDuration?: number; // seconds + frameRate: number; + codec: string; + hasAudio: boolean; + audioCodec?: string; + audioSampleRate?: number; +} + +export interface VideoDecodeFailureContext { + decoderConfig: VideoDecoderConfig; + sourceMetadata?: DecodedVideoInfo; + chunkIndex?: number; + chunk?: EncodedVideoChunk; + decoderState?: CodecState; + decodeQueueSize?: number; +} + +/** Maps WebCodecs failures to stable support-facing identifiers. */ +export function getVideoDecodeFailureCode(error: unknown): string { + const name = error instanceof DOMException ? error.name : ""; + switch (name) { + case "EncodingError": + return "VIDEO_DECODE_ENCODING_ERROR"; + case "NotSupportedError": + return "VIDEO_CODEC_UNSUPPORTED"; + case "QuotaExceededError": + return "VIDEO_DECODER_RESOURCE_EXHAUSTED"; + case "InvalidStateError": + return "VIDEO_DECODER_INVALID_STATE"; + default: + return "VIDEO_DECODE_FAILED"; + } +} + +function describeUnknownError(error: unknown): string { + if (error instanceof DOMException) { + return `${error.name}: ${error.message}`; + } + + if (error instanceof Error) { + return error.message; + } + + return String(error); +} + +/** Builds a decode error with codec, source, chunk, and decoder-state context. */ +export function buildVideoDecodeFailure(error: unknown, context: VideoDecodeFailureContext): Error { + const details = [`codec=${context.decoderConfig.codec}`]; + const failureCode = getVideoDecodeFailureCode(error); + const width = context.decoderConfig.codedWidth; + const height = context.decoderConfig.codedHeight; + if (width && height) { + details.push(`codedSize=${width}x${height}`); + } + if (context.decoderConfig.hardwareAcceleration) { + details.push(`hardwareAcceleration=${context.decoderConfig.hardwareAcceleration}`); + } + if (context.sourceMetadata) { + details.push(`sourceFps=${context.sourceMetadata.frameRate}`); + details.push(`sourceDurationSec=${context.sourceMetadata.duration}`); + } + if (context.chunkIndex !== undefined) { + details.push(`chunkIndex=${context.chunkIndex}`); + } + if (context.chunk) { + details.push(`chunkType=${context.chunk.type}`); + details.push(`chunkTimestampUs=${context.chunk.timestamp}`); + details.push(`sourceTimeSec=${(context.chunk.timestamp / 1_000_000).toFixed(3)}`); + if (typeof context.chunk.duration === "number") { + details.push(`chunkDurationUs=${context.chunk.duration}`); + } + details.push(`chunkBytes=${context.chunk.byteLength}`); + } + if (context.decoderState) { + details.push(`decoderState=${context.decoderState}`); + } + if (context.decodeQueueSize !== undefined) { + details.push(`decodeQueueSize=${context.decodeQueueSize}`); + } + + const failure = new Error( + `[${failureCode}] VideoDecoder failure: ${describeUnknownError(error)} (${details.join(", ")})`, + ); + (failure as Error & { cause?: unknown }).cause = error; + return failure; +} + +/** Keeps the original decoder failure when cleanup triggers secondary errors. */ +export function preserveFirstVideoDecodeFailure( + existingError: Error | null, + error: unknown, + context: VideoDecodeFailureContext, +): Error { + return existingError ?? buildVideoDecodeFailure(error, context); +} + +export function getDecodedFrameStartupOffsetUs( + firstDecodedFrameTimestampUs: number, + metadata: Pick, +): number { + const streamStartTimeUs = Math.round( + (metadata.streamStartTime ?? metadata.mediaStartTime ?? 0) * 1_000_000, + ); + + return Math.max(0, firstDecodedFrameTimestampUs - streamStartTimeUs); +} + +export function getDecodedFrameTimelineOffsetUs( + firstDecodedFrameTimestampUs: number, + metadata: Pick, +): number { + const mediaStartTimeUs = Math.round((metadata.mediaStartTime ?? 0) * 1_000_000); + const streamStartTimeUs = Math.round( + (metadata.streamStartTime ?? metadata.mediaStartTime ?? 0) * 1_000_000, + ); + + return ( + Math.max(0, streamStartTimeUs - mediaStartTimeUs) + + getDecodedFrameStartupOffsetUs(firstDecodedFrameTimestampUs, metadata) + ); +} diff --git a/src/lib/exporter/videoTimelineSegments.ts b/src/lib/exporter/videoTimelineSegments.ts new file mode 100644 index 00000000..9cb19822 --- /dev/null +++ b/src/lib/exporter/videoTimelineSegments.ts @@ -0,0 +1,72 @@ +import type { SpeedRegion, TrimRegion } from "@/components/video-editor/types"; + +export function computeVideoSegments( + totalDuration: number, + trimRegions?: TrimRegion[], +): Array<{ startSec: number; endSec: number }> { + if (!trimRegions || trimRegions.length === 0) { + return [{ startSec: 0, endSec: totalDuration }]; + } + + const sorted = [...trimRegions].sort((a, b) => a.startMs - b.startMs); + const segments: Array<{ startSec: number; endSec: number }> = []; + let cursor = 0; + + for (const trim of sorted) { + const trimStart = trim.startMs / 1000; + const trimEnd = trim.endMs / 1000; + if (cursor < trimStart) { + segments.push({ startSec: cursor, endSec: trimStart }); + } + cursor = Math.max(cursor, trimEnd); + } + + if (cursor < totalDuration) { + segments.push({ startSec: cursor, endSec: totalDuration }); + } + + return segments; +} + +export function splitVideoSegmentsBySpeed( + segments: Array<{ startSec: number; endSec: number }>, + speedRegions?: SpeedRegion[], +): Array<{ startSec: number; endSec: number; speed: number }> { + if (!speedRegions || speedRegions.length === 0) + return segments.map((s) => ({ ...s, speed: 1 })); + + const result: Array<{ startSec: number; endSec: number; speed: number }> = []; + for (const segment of segments) { + const overlapping = speedRegions + .filter( + (sr) => sr.startMs / 1000 < segment.endSec && sr.endMs / 1000 > segment.startSec, + ) + .sort((a, b) => a.startMs - b.startMs); + + if (overlapping.length === 0) { + result.push({ ...segment, speed: 1 }); + continue; + } + + let cursor = segment.startSec; + for (const sr of overlapping) { + const srStart = Math.max(sr.startMs / 1000, segment.startSec); + const srEnd = Math.min(sr.endMs / 1000, segment.endSec); + if (cursor < srStart) { + result.push({ startSec: cursor, endSec: srStart, speed: 1 }); + } + const effectiveStart = Math.max(cursor, srStart); + if (srEnd > effectiveStart) { + result.push({ + startSec: effectiveStart, + endSec: srEnd, + speed: sr.speed, + }); + } + cursor = Math.max(cursor, srEnd); + } + if (cursor < segment.endSec) + result.push({ startSec: cursor, endSec: segment.endSec, speed: 1 }); + } + return result.filter((s) => s.endSec - s.startSec > 0.0001); +}