import { WebDemuxer } from "web-demuxer"; import { SOURCE_AUDIO_NORMALIZE_GAIN } from "@/components/video-editor/audio/audioTypes"; import type { AudioRegion, ClipRegion, SourceAudioTrackSettings, SpeedRegion, TrimRegion, } from "@/components/video-editor/types"; import { buildResolvedAudioPlan, SourceTrackId } from "@/lib/exporter/audioRoutingEngine"; import { estimateCompanionAudioStartDelaySeconds } from "@/lib/mediaTiming"; import { resolveMediaElementSource } from "./localMediaSource"; import type { VideoMuxer } from "./muxer"; import { resolveSourceTrackRoutingPolicy } from "./sourceTrackRoutingPolicy"; const AUDIO_BITRATE = 128_000; const DECODE_BACKPRESSURE_LIMIT = 20; const ENCODE_BACKPRESSURE_LIMIT = 20; const MIN_SPEED_REGION_DELTA_MS = 0.0001; const MP4_AUDIO_CODEC = "mp4a.40.2"; const OFFLINE_AUDIO_SAMPLE_RATE = 48_000; const OFFLINE_ENCODE_CHUNK_FRAMES = 1024; const OFFLINE_CHUNK_DURATION_SEC = 30; function resolveSourceTrackGain( sourceAudioTrackSettings: SourceAudioTrackSettings | undefined, trackId: "mic" | "system" | "mixed", ) { const settings = sourceAudioTrackSettings?.[trackId]; if (!settings) { return 1; } const normalizeGain = settings.normalize ? SOURCE_AUDIO_NORMALIZE_GAIN : 1; return Math.max(0, Math.min(2, settings.volume * normalizeGain)); } export function getSourceTrackIdFromPath(audioPath: string): SourceTrackId { const normalized = audioPath.toLowerCase(); // Check for common patterns like .mic., -mic., mic.mp4, etc. if ( normalized.includes(".mic.") || normalized.includes("-mic.") || normalized.includes("_mic_") || normalized.includes("/mic.") || normalized.includes("\\mic.") || normalized.endsWith("mic.mp4") || normalized.endsWith("mic.m4a") || normalized.endsWith("mic.wav") ) { return "mic"; } if ( normalized.includes(".system.") || normalized.includes("-system.") || normalized.includes("_system_") || normalized.includes("/system.") || normalized.includes("\\system.") || normalized.endsWith("system.mp4") || normalized.endsWith("system.m4a") || normalized.endsWith("system.wav") ) { return "system"; } return "mixed"; } export function hasNonDefaultSourceTrackSettings( sourceAudioTrackSettings?: SourceAudioTrackSettings, ) { if (!sourceAudioTrackSettings) { return false; } return Object.values(sourceAudioTrackSettings).some( (settings) => Math.abs((settings?.volume ?? 1) - 1) > 0.0005 || Boolean(settings?.normalize), ); } interface TimelineSlice { sourceStartMs: number; sourceEndMs: number; speed: number; } interface PreparedOfflineRender { mainBufferEntry: { buffer: AudioBuffer; gain: number } | null; companionEntries: Array<{ buffer: AudioBuffer; startDelaySec: number; gain: number }>; regionEntries: Array<{ buffer: AudioBuffer; region: AudioRegion }>; mutedSourceOutputRangesSec: Array<{ startSec: number; endSec: number }>; slices: TimelineSlice[]; outputDurationMs: number; numChannels: number; } export async function isAacAudioEncodingSupported( sampleRate = 48_000, numberOfChannels = 2, ): Promise { try { const support = await AudioEncoder.isConfigSupported({ codec: MP4_AUDIO_CODEC, sampleRate, numberOfChannels, bitrate: AUDIO_BITRATE, }); return support.supported === true; } catch { return false; } } type TrimLikeRegion = TrimRegion | ClipRegion; export class AudioProcessor { private cancelled = false; private onProgress?: (progress: number) => void; private isPassthroughAudioCodec(codec: string | undefined): boolean { if (!codec) { return false; } const normalizedCodec = codec.toLowerCase(); return ( normalizedCodec === MP4_AUDIO_CODEC || normalizedCodec === "aac" || normalizedCodec.startsWith("mp4a.40.2") ); } private async passthroughAudioStream( audioStream: ReadableStream, audioConfig: AudioDecoderConfig, muxer: VideoMuxer, ): Promise { if (!this.isPassthroughAudioCodec(audioConfig.codec)) { return false; } let reader: ReadableStreamDefaultReader | null = null; let wroteAudio = false; let passthroughTimestampOffsetUs: number | null = null; try { reader = audioStream.getReader(); while (!this.cancelled) { const { done, value: chunk } = await reader.read(); if (done || !chunk) break; if (passthroughTimestampOffsetUs === null) { passthroughTimestampOffsetUs = chunk.timestamp; } const normalizedTimestamp = Math.max( 0, chunk.timestamp - passthroughTimestampOffsetUs, ); const outputChunk = passthroughTimestampOffsetUs === 0 ? chunk : this.cloneEncodedAudioChunkWithTimestamp(chunk, normalizedTimestamp); await muxer.addAudioChunk( outputChunk, wroteAudio ? undefined : { decoderConfig: audioConfig, }, ); wroteAudio = true; } } finally { if (reader) { try { await reader.cancel(); } catch { // reader already closed } } } return wroteAudio; } /** * Audio export has two modes: * 1) no speed regions -> fast WebCodecs trim-only pipeline * 2) speed regions present -> pitch-preserving rendered timeline pipeline */ setOnProgress(callback: (progress: number) => void) { this.onProgress = callback; } async process( demuxer: WebDemuxer | null, muxer: VideoMuxer, videoUrl: string, trimRegions?: TrimLikeRegion[], speedRegions?: SpeedRegion[], readEndSec?: number, audioRegions?: AudioRegion[], sourceAudioFallbackPaths?: string[], sourceAudioFallbackStartDelayMsByPath?: Record, sourceAudioTrackSettings?: SourceAudioTrackSettings, clipRegions?: ClipRegion[], ): Promise { const sortedTrims = trimRegions ? [...trimRegions].sort((a, b) => a.startMs - b.startMs) : []; const sortedSpeedRegions = speedRegions ? [...speedRegions] .filter((region) => region.endMs - region.startMs > MIN_SPEED_REGION_DELTA_MS) .sort((a, b) => a.startMs - b.startMs) : []; const sortedAudioRegions = audioRegions ? [...audioRegions].sort((a, b) => a.startMs - b.startMs) : []; const sortedSourceAudioFallbackPaths = sourceAudioFallbackPaths ? sourceAudioFallbackPaths.filter( (audioPath) => typeof audioPath === "string" && audioPath.trim().length > 0, ) : []; const routingPolicy = resolveSourceTrackRoutingPolicy( videoUrl, sortedSourceAudioFallbackPaths, ); const hasTimedCompanionAudio = routingPolicy.playbackPaths.some( (audioPath) => (sourceAudioFallbackStartDelayMsByPath?.[audioPath] ?? 0) > 0, ); const needsSourceAudioMixing = routingPolicy.playbackPaths.length > 1 || (routingPolicy.hasEmbeddedSourceAudio && routingPolicy.playbackPaths.length > 0) || hasTimedCompanionAudio; // When speed edits, audio regions, or multiple audio sources need mixing, use offline AudioContext pipeline. if ( sortedSpeedRegions.length > 0 || sortedAudioRegions.length > 0 || needsSourceAudioMixing || hasNonDefaultSourceTrackSettings(sourceAudioTrackSettings) || (clipRegions ?? []).some((clip) => Boolean(clip.muted)) ) { await this.renderAndMuxOfflineAudio( videoUrl, sortedTrims, sortedSpeedRegions, sortedAudioRegions, sortedSourceAudioFallbackPaths, sourceAudioFallbackStartDelayMsByPath, sourceAudioTrackSettings, clipRegions, muxer, ); return; } // Single sidecar audio with no speed/audio edits: demux directly (skips slow real-time rendering). if (!routingPolicy.hasEmbeddedSourceAudio && routingPolicy.playbackPaths.length === 1) { const sidecarDemuxer = await this.loadAudioFileDemuxer(routingPolicy.playbackPaths[0]); if (sidecarDemuxer) { try { await this.processTrimOnlyAudio(sidecarDemuxer, muxer, sortedTrims); } finally { try { sidecarDemuxer.destroy(); } catch { /* cleanup */ } } return; } // Fallback to offline rendering if demuxer creation failed console.warn( "[AudioProcessor] Fast sidecar demux failed, falling back to offline rendering", ); await this.renderAndMuxOfflineAudio( videoUrl, sortedTrims, [], [], routingPolicy.playbackPaths, sourceAudioFallbackStartDelayMsByPath, sourceAudioTrackSettings, clipRegions, muxer, ); return; } // No speed edits or audio regions: keep the original demux/decode/encode path with trim timestamp remap. if (!demuxer) { console.warn("[AudioProcessor] No demuxer available, skipping audio"); return; } if (sortedTrims.length === 0) { let audioConfig: AudioDecoderConfig; try { audioConfig = (await demuxer.getDecoderConfig("audio")) as AudioDecoderConfig; } catch { console.warn("[AudioProcessor] No audio track found, skipping"); return; } const audioStream = typeof readEndSec === "number" ? demuxer.read("audio", 0, readEndSec) : demuxer.read("audio"); const copiedSourceAudio = await this.passthroughAudioStream( audioStream as ReadableStream, audioConfig, muxer, ); if (copiedSourceAudio) { return; } } await this.processTrimOnlyAudio(demuxer, muxer, sortedTrims, readEndSec); } async renderEditedAudioTrack( videoUrl: string, trimRegions?: TrimLikeRegion[], speedRegions?: SpeedRegion[], audioRegions?: AudioRegion[], sourceAudioFallbackPaths?: string[], sourceAudioFallbackStartDelayMsByPath?: Record, sourceAudioTrackSettings?: SourceAudioTrackSettings, clipRegions?: ClipRegion[], ): Promise { const sortedTrims = trimRegions ? [...trimRegions].sort((a, b) => a.startMs - b.startMs) : []; const sortedSpeedRegions = speedRegions ? [...speedRegions] .filter((region) => region.endMs - region.startMs > MIN_SPEED_REGION_DELTA_MS) .sort((a, b) => a.startMs - b.startMs) : []; const sortedAudioRegions = audioRegions ? [...audioRegions].sort((a, b) => a.startMs - b.startMs) : []; const sortedSourceAudioFallbackPaths = sourceAudioFallbackPaths ? sourceAudioFallbackPaths.filter( (audioPath) => typeof audioPath === "string" && audioPath.trim().length > 0, ) : []; const prepared = await this.prepareOfflineRender( videoUrl, sortedTrims, sortedSpeedRegions, sortedAudioRegions, sortedSourceAudioFallbackPaths, sourceAudioFallbackStartDelayMsByPath, sourceAudioTrackSettings, clipRegions, ); return this.renderToWavBlobChunked(prepared); } // Legacy trim-only path used when no speed regions are configured. private async processTrimOnlyAudio( demuxer: WebDemuxer, muxer: VideoMuxer, sortedTrims: TrimLikeRegion[], readEndSec?: number, ): Promise { let audioConfig: AudioDecoderConfig; try { audioConfig = (await demuxer.getDecoderConfig("audio")) as AudioDecoderConfig; } catch { console.warn("[AudioProcessor] No audio track found, skipping"); return; } const codecCheck = await AudioDecoder.isConfigSupported(audioConfig); if (!codecCheck.supported) { console.warn("[AudioProcessor] Audio codec not supported:", audioConfig.codec); return; } const audioStream = typeof readEndSec === "number" ? demuxer.read("audio", 0, readEndSec) : demuxer.read("audio"); let sourceTimestampOffsetUs: number | null = null; await this.transcodeAudioStream( audioStream as ReadableStream, audioConfig, muxer, { observeChunkTimestampUs: (timestampUs) => { if (sourceTimestampOffsetUs === null) { sourceTimestampOffsetUs = timestampUs; } }, shouldSkipChunk: (timestampMs) => this.isInTrimRegion(timestampMs, sortedTrims), transformAudioData: (data) => { const timestampMs = data.timestamp / 1000; const trimOffsetMs = this.computeTrimOffset(timestampMs, sortedTrims); const adjustedTimestampUs = data.timestamp - (sourceTimestampOffsetUs ?? 0) - trimOffsetMs * 1000; return this.cloneWithTimestamp(data, Math.max(0, adjustedTimestampUs)); }, }, ); } private async transcodeAudioStream( audioStream: ReadableStream, audioConfig: AudioDecoderConfig, muxer: VideoMuxer, options: { observeChunkTimestampUs?: (timestampUs: number) => void; shouldSkipChunk?: (timestampMs: number) => boolean; transformAudioData?: (data: AudioData) => AudioData | null; } = {}, ): Promise { const pendingFrames: AudioData[] = []; let decodeError: Error | null = null; let encodeError: Error | null = null; let muxError: Error | null = null; let pendingMuxing = Promise.resolve(); const capacityWaiters = new Set<() => void>(); const notifyCapacityAvailable = () => { if (capacityWaiters.size === 0) { return; } const waiters = [...capacityWaiters]; capacityWaiters.clear(); for (const resolve of waiters) { resolve(); } }; const waitForCapacity = () => new Promise((resolve) => { capacityWaiters.add(resolve); }); const failIfNeeded = () => { if (decodeError) throw decodeError; if (encodeError) throw encodeError; if (muxError) throw muxError; }; const pumpEncodedFrames = () => { while (!this.cancelled && pendingFrames.length > 0) { if (encodeError || muxError) { break; } if (encoder.encodeQueueSize >= ENCODE_BACKPRESSURE_LIMIT) { break; } const frame = pendingFrames.shift(); if (!frame) { break; } encoder.encode(frame); frame.close(); notifyCapacityAvailable(); } }; const cleanupPendingFrames = () => { for (const frame of pendingFrames) { frame.close(); } pendingFrames.length = 0; }; const sampleRate = audioConfig.sampleRate || 48_000; const channels = audioConfig.numberOfChannels || 2; const encodeConfig: AudioEncoderConfig = { codec: MP4_AUDIO_CODEC, sampleRate, numberOfChannels: channels, bitrate: AUDIO_BITRATE, }; const encodeSupport = await AudioEncoder.isConfigSupported(encodeConfig); if (!encodeSupport.supported) { console.warn("[AudioProcessor] AAC encoding not supported, skipping audio"); return; } const encoder = new AudioEncoder({ output: (chunk: EncodedAudioChunk, meta?: EncodedAudioChunkMetadata) => { pendingMuxing = pendingMuxing .then(async () => { if (this.cancelled) { return; } await muxer.addAudioChunk(chunk, meta); }) .catch((error) => { muxError = error instanceof Error ? error : new Error(String(error)); notifyCapacityAvailable(); }); notifyCapacityAvailable(); }, error: (error: DOMException) => { encodeError = new Error(`[AudioProcessor] Encode error: ${error.message}`); notifyCapacityAvailable(); }, }); encoder.configure(encodeConfig); const decoder = new AudioDecoder({ output: (data: AudioData) => { if (this.cancelled || encodeError || muxError) { data.close(); return; } const transformed = options.transformAudioData ? options.transformAudioData(data) : data; if (transformed !== data) { data.close(); } if (!transformed) { return; } pendingFrames.push(transformed); notifyCapacityAvailable(); }, error: (error: DOMException) => { decodeError = new Error(`[AudioProcessor] Decode error: ${error.message}`); notifyCapacityAvailable(); }, }); decoder.configure(audioConfig); let reader: ReadableStreamDefaultReader | null = null; try { reader = audioStream.getReader(); while (!this.cancelled) { failIfNeeded(); const { done, value: chunk } = await reader.read(); if (done || !chunk) break; options.observeChunkTimestampUs?.(chunk.timestamp); const timestampMs = chunk.timestamp / 1000; if (options.shouldSkipChunk?.(timestampMs)) continue; decoder.decode(chunk); pumpEncodedFrames(); while ( !this.cancelled && (decoder.decodeQueueSize > DECODE_BACKPRESSURE_LIMIT || pendingFrames.length > DECODE_BACKPRESSURE_LIMIT || encoder.encodeQueueSize >= ENCODE_BACKPRESSURE_LIMIT) ) { failIfNeeded(); pumpEncodedFrames(); await waitForCapacity(); } } if (decoder.state === "configured") { await decoder.flush(); } while (!this.cancelled && (pendingFrames.length > 0 || encoder.encodeQueueSize > 0)) { failIfNeeded(); pumpEncodedFrames(); if (pendingFrames.length > 0 || encoder.encodeQueueSize > 0) { await waitForCapacity(); } } failIfNeeded(); if (encoder.state === "configured") { await encoder.flush(); } await pendingMuxing; failIfNeeded(); } finally { notifyCapacityAvailable(); if (reader) { try { await reader.cancel(); } catch { // reader already closed } } cleanupPendingFrames(); if (decoder.state === "configured") { decoder.close(); } if (encoder.state === "configured") { encoder.close(); } } if (this.cancelled) { return; } } // ---------- Offline audio rendering pipeline ---------- // Replaces the old real-time MediaElement+MediaRecorder approach with // OfflineAudioContext, which renders as fast as the CPU allows instead of // waiting for 1× real-time playback. private async renderAndMuxOfflineAudio( videoUrl: string, trimRegions: TrimLikeRegion[], speedRegions: SpeedRegion[], audioRegions: AudioRegion[], sourceAudioFallbackPaths: string[], sourceAudioFallbackStartDelayMsByPath: Record | undefined, sourceAudioTrackSettings: SourceAudioTrackSettings | undefined, clipRegions: ClipRegion[] | undefined, muxer: VideoMuxer, ): Promise { const prepared = await this.prepareOfflineRender( videoUrl, trimRegions, speedRegions, audioRegions, sourceAudioFallbackPaths, sourceAudioFallbackStartDelayMsByPath, sourceAudioTrackSettings, clipRegions, ); if (this.cancelled) return; await this.renderAndEncodeChunked(prepared, muxer); } private async prepareOfflineRender( videoUrl: string, trimRegions: TrimLikeRegion[], speedRegions: SpeedRegion[], audioRegions: AudioRegion[], sourceAudioFallbackPaths: string[], sourceAudioFallbackStartDelayMsByPath?: Record, sourceAudioTrackSettings?: SourceAudioTrackSettings, clipRegions?: ClipRegion[], ): Promise { if (this.cancelled) throw new Error("Export cancelled"); this.onProgress?.(0); const resolvedPlan = buildResolvedAudioPlan({ videoResource: videoUrl, sourceAudioFallbackPaths, audioRegions, sourceTrackGainById: { mic: resolveSourceTrackGain(sourceAudioTrackSettings, "mic"), system: resolveSourceTrackGain(sourceAudioTrackSettings, "system"), mixed: resolveSourceTrackGain(sourceAudioTrackSettings, "mixed"), }, embeddedGain: Math.max( 0, Math.min( 2, sourceAudioTrackSettings?.mixed ? resolveSourceTrackGain(sourceAudioTrackSettings, "mixed") : sourceAudioTrackSettings?.system ? resolveSourceTrackGain(sourceAudioTrackSettings, "system") : 1, ), ), }); // Decode embedded source audio separately from companion sidecars. const mainBuffer = resolvedPlan.includeEmbeddedInExport ? await this.decodeAudioFromUrl(videoUrl) : null; const mainBufferGain = resolveSourceTrackGain(sourceAudioTrackSettings, "mixed"); const mainBufferEntry = mainBuffer ? { buffer: mainBuffer, gain: mainBufferGain } : null; if (this.cancelled) throw new Error("Export cancelled"); // Decode companion / sidecar audio files const companionEntries: Array<{ buffer: AudioBuffer; startDelaySec: number; gain: number; }> = []; const refDuration = mainBuffer?.duration ?? (resolvedPlan.playbackPaths.length > 0 ? await this.getMediaDurationSec(videoUrl) : 0); for (const audioPath of resolvedPlan.playbackPaths) { if (this.cancelled) throw new Error("Export cancelled"); const buffer = await this.decodeAudioFromUrl(audioPath); if (!buffer) continue; companionEntries.push({ buffer, gain: resolveSourceTrackGain( sourceAudioTrackSettings, getSourceTrackIdFromPath(audioPath), ), startDelaySec: estimateCompanionAudioStartDelaySeconds( refDuration, buffer.duration, sourceAudioFallbackStartDelayMsByPath?.[audioPath], ), }); } if (this.cancelled) throw new Error("Export cancelled"); // Decode audio region overlay files const regionEntries: Array<{ buffer: AudioBuffer; region: AudioRegion }> = []; for (const region of audioRegions) { if (this.cancelled) throw new Error("Export cancelled"); const buffer = await this.decodeAudioFromUrl(region.audioPath); if (buffer) regionEntries.push({ buffer, region }); } this.onProgress?.(0.2); // Determine source duration for timeline calculation const primaryBuffer = mainBufferEntry?.buffer ?? companionEntries[0]?.buffer ?? null; if (!primaryBuffer && regionEntries.length === 0) { throw new Error("No decodable audio sources found"); } let sourceDurationSec: number; if (mainBufferEntry?.buffer) { sourceDurationSec = mainBufferEntry.buffer.duration; } else if (resolvedPlan.playbackPaths.length > 0 || regionEntries.length > 0) { sourceDurationSec = await this.getMediaDurationSec(videoUrl); } else { sourceDurationSec = primaryBuffer?.duration ?? 0; } const sourceDurationMs = sourceDurationSec * 1000; // Build timeline slices (non-trimmed segments with speed info) const slices = this.buildTimelineSlices(sourceDurationMs, trimRegions, speedRegions); let outputDurationMs = 0; for (const slice of slices) { outputDurationMs += (slice.sourceEndMs - slice.sourceStartMs) / slice.speed; } // Extend for audio regions that might exceed the video timeline for (const { region } of regionEntries) { const regionEndOutput = this.sourceTimeToOutputTime(region.endMs, slices); outputDurationMs = Math.max(outputDurationMs, regionEndOutput); } const numChannels = Math.min(primaryBuffer?.numberOfChannels ?? 2, 2); const mutedSourceOutputRangesSec = (clipRegions ?? []) .filter( (clip) => Boolean(clip.muted) && Number.isFinite(clip.startMs) && Number.isFinite(clip.endMs) && clip.endMs > clip.startMs, ) .map((clip) => ({ startSec: Math.max(0, clip.startMs / 1000), endSec: Math.max(0, clip.endMs / 1000), })); return { mainBufferEntry, companionEntries, regionEntries, mutedSourceOutputRangesSec, slices, outputDurationMs, numChannels, }; } // Render timeline in chunks and encode each chunk to the muxer immediately. // Memory is bounded to ~OFFLINE_CHUNK_DURATION_SEC of PCM per chunk // instead of holding the entire output buffer in memory. private async renderAndEncodeChunked( prepared: PreparedOfflineRender, muxer: VideoMuxer, ): Promise { const { numChannels } = prepared; const totalOutputSec = Math.max(prepared.outputDurationMs / 1000, 0.01); let encodeError: Error | null = null; let muxError: Error | null = null; let pendingMuxing = Promise.resolve(); let wroteFirstChunk = false; const encodeConfig: AudioEncoderConfig = { codec: MP4_AUDIO_CODEC, sampleRate: OFFLINE_AUDIO_SAMPLE_RATE, numberOfChannels: numChannels, bitrate: AUDIO_BITRATE, }; const supported = await AudioEncoder.isConfigSupported(encodeConfig); if (!supported.supported) { console.warn("[AudioProcessor] AAC encoding not supported for offline audio"); return; } const encoder = new AudioEncoder({ output: (chunk: EncodedAudioChunk, meta?: EncodedAudioChunkMetadata) => { pendingMuxing = pendingMuxing .then(async () => { if (this.cancelled) return; await muxer.addAudioChunk(chunk, !wroteFirstChunk ? meta : undefined); wroteFirstChunk = true; }) .catch((error) => { muxError = error instanceof Error ? error : new Error(String(error)); }); }, error: (error: DOMException) => { encodeError = new Error(`Audio encode error: ${error.message}`); }, }); encoder.configure(encodeConfig); try { await this.renderChunked( prepared, totalOutputSec, async (rendered, outputOffsetSec) => { if (encodeError) throw encodeError; if (muxError) throw muxError; await this.feedBufferToEncoder(encoder, rendered, outputOffsetSec); }, ); if (encodeError) throw encodeError; if (muxError) throw muxError; if (encoder.state === "configured") { await encoder.flush(); } await pendingMuxing; if (encodeError) throw encodeError; if (muxError) throw muxError; } finally { if (encoder.state === "configured") { encoder.close(); } } } // Render timeline to a WAV blob for the native/FFmpeg export path. // Processes in chunks to avoid holding the entire output in memory. private async renderToWavBlobChunked(prepared: PreparedOfflineRender): Promise { const totalOutputSec = Math.max(prepared.outputDurationMs / 1000, 0.01); const totalFrames = Math.ceil(totalOutputSec * OFFLINE_AUDIO_SAMPLE_RATE); const numChannels = prepared.numChannels; const header = this.createWavHeader(OFFLINE_AUDIO_SAMPLE_RATE, numChannels, totalFrames); const pcmParts: ArrayBuffer[] = [header]; await this.renderChunked(prepared, totalOutputSec, async (rendered) => { pcmParts.push(...this.audioBufferToPcmParts(rendered)); }); return new Blob(pcmParts, { type: "audio/wav" }); } // Shared chunked rendering loop. Processes the timeline in // OFFLINE_CHUNK_DURATION_SEC segments, calling onChunk for each rendered buffer. private async renderChunked( prepared: PreparedOfflineRender, totalOutputSec: number, onChunk: ( rendered: AudioBuffer, outputOffsetSec: number, chunkIndex: number, ) => Promise, ): Promise { const { slices, numChannels } = prepared; let outputOffsetSec = 0; const chunkCount = Math.ceil(totalOutputSec / OFFLINE_CHUNK_DURATION_SEC); for (let i = 0; i < chunkCount && !this.cancelled; i++) { const chunkSec = Math.min(OFFLINE_CHUNK_DURATION_SEC, totalOutputSec - outputOffsetSec); const chunkFrames = Math.ceil(chunkSec * OFFLINE_AUDIO_SAMPLE_RATE); const offlineCtx = new OfflineAudioContext( numChannels, chunkFrames, OFFLINE_AUDIO_SAMPLE_RATE, ); // Schedule main audio if (prepared.mainBufferEntry) { this.scheduleBufferThroughTimeline( offlineCtx, prepared.mainBufferEntry.buffer, slices, 0, prepared.mainBufferEntry.gain, outputOffsetSec, chunkSec, prepared.mutedSourceOutputRangesSec, ); } // Schedule companion/sidecar audio for (const entry of prepared.companionEntries) { this.scheduleBufferThroughTimeline( offlineCtx, entry.buffer, slices, entry.startDelaySec, entry.gain, outputOffsetSec, chunkSec, prepared.mutedSourceOutputRangesSec, ); } // Schedule audio region overlays for (const { buffer, region } of prepared.regionEntries) { this.scheduleRegionForChunk( offlineCtx, buffer, region, slices, outputOffsetSec, chunkSec, ); } const rendered = await offlineCtx.startRendering(); if (this.cancelled) break; await onChunk(rendered, outputOffsetSec, i); outputOffsetSec += chunkSec; this.onProgress?.(0.3 + (outputOffsetSec / totalOutputSec) * 0.7); } } // Schedule an audio region overlay clipped to a specific chunk window. private scheduleRegionForChunk( ctx: OfflineAudioContext, buffer: AudioBuffer, region: AudioRegion, slices: TimelineSlice[], chunkOutputStartSec: number, chunkDurationSec: number, ): void { const outputStartMs = this.sourceTimeToOutputTime(region.startMs, slices); const outputEndMs = this.sourceTimeToOutputTime(region.endMs, slices); let localStartSec = outputStartMs / 1000 - chunkOutputStartSec; let localEndSec = outputEndMs / 1000 - chunkOutputStartSec; // Skip if region doesn't overlap with this chunk if (localEndSec <= 0 || localStartSec >= chunkDurationSec) return; // Clip to chunk bounds let bufferOffsetSec = 0; if (localStartSec < 0) { bufferOffsetSec = -localStartSec; localStartSec = 0; } if (localEndSec > chunkDurationSec) { localEndSec = chunkDurationSec; } const duration = Math.min(localEndSec - localStartSec, buffer.duration - bufferOffsetSec); if (duration <= 0.001) return; const gainNode = ctx.createGain(); const normalizeGain = region.normalize ? SOURCE_AUDIO_NORMALIZE_GAIN : 1; gainNode.gain.value = Math.max(0, Math.min(1, region.volume * normalizeGain)); gainNode.connect(ctx.destination); const source = ctx.createBufferSource(); source.buffer = buffer; source.connect(gainNode); source.start(localStartSec, bufferOffsetSec, duration); } // Feed a rendered AudioBuffer chunk to an AudioEncoder with a timestamp offset. private async feedBufferToEncoder( encoder: AudioEncoder, buffer: AudioBuffer, timestampOffsetSec: number, ): Promise { const sampleRate = buffer.sampleRate; const numChannels = buffer.numberOfChannels; const totalFrames = buffer.length; for ( let offset = 0; offset < totalFrames && !this.cancelled; offset += OFFLINE_ENCODE_CHUNK_FRAMES ) { const frameCount = Math.min(OFFLINE_ENCODE_CHUNK_FRAMES, totalFrames - offset); const planarData = new Float32Array(frameCount * numChannels); for (let ch = 0; ch < numChannels; ch++) { const channelData = buffer.getChannelData(ch); planarData.set(channelData.subarray(offset, offset + frameCount), ch * frameCount); } const audioData = new AudioData({ format: "f32-planar", sampleRate, numberOfFrames: frameCount, numberOfChannels: numChannels, timestamp: Math.round((offset / sampleRate + timestampOffsetSec) * 1_000_000), data: planarData, }); encoder.encode(audioData); audioData.close(); while (encoder.encodeQueueSize >= ENCODE_BACKPRESSURE_LIMIT && !this.cancelled) { await new Promise((r) => setTimeout(r, 1)); } } } // Decode audio from a URL using streaming WebCodecs decode with bulk fallback. // Streaming decode avoids holding the full compressed file in memory alongside // the decoded AudioBuffer, reducing peak memory for large recordings. private async decodeAudioFromUrl(url: string): Promise { try { const buffer = await this.streamDecodeFromUrl(url); if (buffer) return buffer; } catch (error) { console.warn( "[AudioProcessor] Streaming decode failed, falling back to bulk decode:", url, error, ); } return this.bulkDecodeFromUrl(url, OFFLINE_AUDIO_SAMPLE_RATE); } // Streaming decode via WebDemuxer + AudioDecoder. Decodes audio chunk-by-chunk // without loading the entire compressed file into a contiguous ArrayBuffer. private async streamDecodeFromUrl(url: string): Promise { const source = await resolveMediaElementSource(url); let demuxer: WebDemuxer | null = null; try { const wasmUrl = new URL("./wasm/web-demuxer.wasm", window.location.href).href; demuxer = new WebDemuxer({ wasmFilePath: wasmUrl }); await demuxer.load(source.src); let audioConfig: AudioDecoderConfig; try { audioConfig = (await demuxer.getDecoderConfig("audio")) as AudioDecoderConfig; } catch { return null; // No audio track } const sampleRate = audioConfig.sampleRate || 48_000; const numChannels = Math.min(audioConfig.numberOfChannels || 2, 2); // Accumulate decoded PCM per channel const channelChunks: Float32Array[][] = Array.from({ length: numChannels }, () => []); let totalFrames = 0; let decodeError: Error | null = null; const decodeCapacityWaiters = new Set<() => void>(); const notifyDecodeCapacityAvailable = () => { if (decodeCapacityWaiters.size === 0) { return; } const waiters = [...decodeCapacityWaiters]; decodeCapacityWaiters.clear(); for (const resolve of waiters) { resolve(); } }; const waitForDecodeCapacity = () => new Promise((resolve) => { decodeCapacityWaiters.add(resolve); }); const decoder = new AudioDecoder({ output: (data: AudioData) => { try { const frames = data.numberOfFrames; const dataChannels = Math.min(data.numberOfChannels, numChannels); const format = data.format; if (format?.includes("planar")) { for (let ch = 0; ch < dataChannels; ch++) { const size = data.allocationSize({ planeIndex: ch, }); const bytes = new ArrayBuffer(size); data.copyTo(bytes, { planeIndex: ch }); channelChunks[ch].push(this.rawToFloat32(bytes, format, frames)); } } else if (format) { // Interleaved format — deinterleave into per-channel arrays. // Use data.numberOfChannels as stride (not capped dataChannels) // since the raw buffer contains all source channels. const srcChannels = data.numberOfChannels; const size = data.allocationSize({ planeIndex: 0 }); const bytes = new ArrayBuffer(size); data.copyTo(bytes, { planeIndex: 0 }); const interleaved = this.rawToFloat32( bytes, format, frames * srcChannels, ); for (let ch = 0; ch < dataChannels; ch++) { const chData = new Float32Array(frames); for (let i = 0; i < frames; i++) { chData[i] = interleaved[i * srcChannels + ch]; } channelChunks[ch].push(chData); } } // Fill missing channels with silence for (let ch = dataChannels; ch < numChannels; ch++) { channelChunks[ch].push(new Float32Array(frames)); } totalFrames += frames; } finally { data.close(); notifyDecodeCapacityAvailable(); } }, error: (err: DOMException) => { decodeError = new Error(`Streaming audio decode error: ${err.message}`); notifyDecodeCapacityAvailable(); }, }); decoder.configure(audioConfig); const audioStream = demuxer.read("audio"); const reader = (audioStream as ReadableStream).getReader(); try { while (!this.cancelled) { if (decodeError) throw decodeError; const { done, value: chunk } = await reader.read(); if (done || !chunk) break; decoder.decode(chunk); while (decoder.decodeQueueSize > DECODE_BACKPRESSURE_LIMIT && !this.cancelled) { if (decodeError) throw decodeError; await waitForDecodeCapacity(); } } if (decoder.state === "configured") { await decoder.flush(); } if (decodeError) throw decodeError; } finally { notifyDecodeCapacityAvailable(); try { await reader.cancel(); } catch { /* reader already closed */ } if (decoder.state === "configured") { decoder.close(); } } if (totalFrames === 0) return null; // Build AudioBuffer from accumulated chunks const audioBuffer = new AudioBuffer({ length: totalFrames, numberOfChannels: numChannels, sampleRate, }); for (let ch = 0; ch < numChannels; ch++) { const channelData = audioBuffer.getChannelData(ch); let writeOffset = 0; for (const chunk of channelChunks[ch]) { channelData.set(chunk, writeOffset); writeOffset += chunk.length; } } return audioBuffer; } finally { source.revoke(); try { demuxer?.destroy(); } catch { /* cleanup */ } } } // Convert raw bytes from AudioData to Float32Array based on the sample format. private rawToFloat32(bytes: ArrayBuffer, format: string, sampleCount: number): Float32Array { if (format.startsWith("f32")) { return new Float32Array(bytes); } if (format.startsWith("s16")) { const int16 = new Int16Array(bytes); const f32 = new Float32Array(sampleCount); for (let i = 0; i < sampleCount; i++) { f32[i] = int16[i] / 0x8000; } return f32; } if (format.startsWith("s32")) { const int32 = new Int32Array(bytes); const f32 = new Float32Array(sampleCount); for (let i = 0; i < sampleCount; i++) { f32[i] = int32[i] / 0x80000000; } return f32; } if (format.startsWith("u8")) { const uint8 = new Uint8Array(bytes); const f32 = new Float32Array(sampleCount); for (let i = 0; i < sampleCount; i++) { f32[i] = (uint8[i] - 128) / 128; } return f32; } // Unknown format — attempt float32 interpretation return new Float32Array(bytes); } // Bulk decode fallback: loads entire file into memory and uses decodeAudioData. private async bulkDecodeFromUrl(url: string, sampleRate: number): Promise { try { const source = await resolveMediaElementSource(url); try { const response = await fetch(source.src); const arrayBuffer = await response.arrayBuffer(); const tempCtx = new OfflineAudioContext(2, 1, sampleRate); return await tempCtx.decodeAudioData(arrayBuffer); } finally { source.revoke(); } } catch (error) { console.warn("[AudioProcessor] Failed to decode audio from URL:", url, error); return null; } } // Get the duration of a media file by loading only its metadata. private async getMediaDurationSec(url: string): Promise { const source = await resolveMediaElementSource(url); try { const media = document.createElement("video"); media.preload = "metadata"; media.src = source.src; return await new Promise((resolve, reject) => { const timeout = setTimeout(() => { cleanup(); media.src = ""; media.load(); reject(new Error("Timed out getting media duration (30s)")); }, 30_000); const onLoaded = () => { cleanup(); const duration = media.duration; media.src = ""; media.load(); resolve(Number.isFinite(duration) ? duration : 0); }; const onError = () => { cleanup(); media.src = ""; media.load(); reject(new Error("Failed to get media duration")); }; const cleanup = () => { clearTimeout(timeout); media.removeEventListener("loadedmetadata", onLoaded); media.removeEventListener("error", onError); }; media.addEventListener("loadedmetadata", onLoaded); media.addEventListener("error", onError, { once: true }); }); } finally { source.revoke(); } } // Build non-overlapping timeline slices from the source timeline, excluding // trimmed regions and tagging each slice with its playback speed. private buildTimelineSlices( sourceDurationMs: number, trimRegions: TrimLikeRegion[], speedRegions: SpeedRegion[], ): TimelineSlice[] { const boundaries = new Set(); boundaries.add(0); boundaries.add(sourceDurationMs); for (const trim of trimRegions) { if (trim.startMs >= 0 && trim.startMs <= sourceDurationMs) boundaries.add(trim.startMs); if (trim.endMs >= 0 && trim.endMs <= sourceDurationMs) boundaries.add(trim.endMs); } for (const speed of speedRegions) { if (speed.startMs >= 0 && speed.startMs <= sourceDurationMs) boundaries.add(speed.startMs); if (speed.endMs >= 0 && speed.endMs <= sourceDurationMs) boundaries.add(speed.endMs); } const sorted = [...boundaries].sort((a, b) => a - b); const slices: TimelineSlice[] = []; for (let i = 0; i < sorted.length - 1; i++) { const start = sorted[i]; const end = sorted[i + 1]; if (end - start < 0.001) continue; // Skip segments entirely inside a trim region const midpoint = (start + end) / 2; if (this.isInTrimRegion(midpoint, trimRegions)) continue; const speedRegion = speedRegions.find( (s) => midpoint >= s.startMs && midpoint < s.endMs, ); slices.push({ sourceStartMs: start, sourceEndMs: end, speed: speedRegion?.speed ?? 1, }); } return slices; } // Map a source-timeline timestamp to the corresponding output-timeline timestamp. private sourceTimeToOutputTime(sourceMs: number, slices: TimelineSlice[]): number { let outputMs = 0; for (const slice of slices) { if (sourceMs <= slice.sourceStartMs) { return outputMs; } const sliceDurationMs = slice.sourceEndMs - slice.sourceStartMs; if (sourceMs >= slice.sourceEndMs) { outputMs += sliceDurationMs / slice.speed; continue; } // Source time falls within this slice outputMs += (sourceMs - slice.sourceStartMs) / slice.speed; return outputMs; } return outputMs; } // Schedule an AudioBuffer through the timeline slices in an OfflineAudioContext. // Each non-trimmed segment creates an AudioBufferSourceNode with the appropriate // playbackRate for speed regions. When chunkOutputStartSec/chunkDurationSec are // provided, only sources overlapping the chunk window are scheduled. private scheduleBufferThroughTimeline( ctx: OfflineAudioContext, buffer: AudioBuffer, slices: TimelineSlice[], sourceStartDelaySec: number, gain = 1, chunkOutputStartSec = 0, chunkDurationSec = Number.POSITIVE_INFINITY, mutedOutputRangesSec: Array<{ startSec: number; endSec: number }> = [], ): void { let outputOffsetSec = 0; for (const slice of slices) { const sliceSourceDurationSec = (slice.sourceEndMs - slice.sourceStartMs) / 1000; const sliceOutputDurationSec = sliceSourceDurationSec / slice.speed; // Where in the buffer does this slice read from? const bufferOffsetSec = slice.sourceStartMs / 1000 - sourceStartDelaySec; // Skip if slice doesn't overlap with the buffer at all if ( bufferOffsetSec + sliceSourceDurationSec <= 0 || bufferOffsetSec >= buffer.duration ) { outputOffsetSec += sliceOutputDurationSec; continue; } // Clamp to buffer bounds let effectiveBufferStartSec = Math.max(0, bufferOffsetSec); const trimmedFromStartSec = effectiveBufferStartSec - bufferOffsetSec; let effectiveSourceDurationSec = Math.min( sliceSourceDurationSec - trimmedFromStartSec, buffer.duration - effectiveBufferStartSec, ); if (effectiveSourceDurationSec <= 0.001) { outputOffsetSec += sliceOutputDurationSec; continue; } // Calculate output position (global then chunk-local) let localOutputStartSec = outputOffsetSec + trimmedFromStartSec / slice.speed - chunkOutputStartSec; let localOutputEndSec = localOutputStartSec + effectiveSourceDurationSec / slice.speed; // Skip if entirely outside chunk window if (localOutputEndSec <= 0 || localOutputStartSec >= chunkDurationSec) { outputOffsetSec += sliceOutputDurationSec; continue; } // Clip to chunk start if (localOutputStartSec < 0) { const skipOutputSec = -localOutputStartSec; const skipSourceSec = skipOutputSec * slice.speed; effectiveBufferStartSec += skipSourceSec; effectiveSourceDurationSec -= skipSourceSec; localOutputStartSec = 0; } // Clip to chunk end if (localOutputEndSec > chunkDurationSec) { const excessOutputSec = localOutputEndSec - chunkDurationSec; effectiveSourceDurationSec -= excessOutputSec * slice.speed; } if (effectiveSourceDurationSec <= 0.001) { outputOffsetSec += sliceOutputDurationSec; continue; } const audibleRanges: Array<{ startSec: number; endSec: number }> = [ { startSec: localOutputStartSec + chunkOutputStartSec, endSec: localOutputStartSec + chunkOutputStartSec + effectiveSourceDurationSec / slice.speed, }, ]; for (const mutedRange of mutedOutputRangesSec) { for (let rangeIndex = audibleRanges.length - 1; rangeIndex >= 0; rangeIndex -= 1) { const current = audibleRanges[rangeIndex]; const overlapStart = Math.max(current.startSec, mutedRange.startSec); const overlapEnd = Math.min(current.endSec, mutedRange.endSec); if (overlapEnd <= overlapStart) { continue; } audibleRanges.splice(rangeIndex, 1); if (current.startSec < overlapStart) { audibleRanges.push({ startSec: current.startSec, endSec: overlapStart }); } if (overlapEnd < current.endSec) { audibleRanges.push({ startSec: overlapEnd, endSec: current.endSec }); } } } for (const audibleRange of audibleRanges) { const audibleDurationSec = audibleRange.endSec - audibleRange.startSec; if (audibleDurationSec <= 0.001) { continue; } const source = ctx.createBufferSource(); const gainNode = ctx.createGain(); gainNode.gain.value = Math.max(0, Math.min(2, gain)); const sourceOffsetSec = effectiveBufferStartSec + (audibleRange.startSec - (localOutputStartSec + chunkOutputStartSec)) * slice.speed; const localStartSec = audibleRange.startSec - chunkOutputStartSec; const sourceDurationSec = audibleDurationSec * slice.speed; const stretchedBuffer = this.stretchAudioBuffer( buffer, slice.speed, sourceOffsetSec, sourceDurationSec, audibleDurationSec, ctx, ); source.buffer = stretchedBuffer; source.playbackRate.value = 1; source.connect(gainNode); gainNode.connect(ctx.destination); source.start(localStartSec); } outputOffsetSec += sliceOutputDurationSec; } } private stretchAudioBuffer( originalBuffer: AudioBuffer, speed: number, sourceOffsetSec: number, sourceDurationSec: number, audibleDurationSec: number, ctx: BaseAudioContext, ): AudioBuffer { const sampleRate = originalBuffer.sampleRate; const channels = originalBuffer.numberOfChannels; const startSample = Math.max(0, Math.floor(sourceOffsetSec * sampleRate)); const sourceSamples = Math.floor(sourceDurationSec * sampleRate); const endSample = Math.min(originalBuffer.length, startSample + sourceSamples); const outSamples = Math.floor(audibleDurationSec * sampleRate); if (outSamples <= 0 || startSample >= originalBuffer.length) { return ctx.createBuffer(channels, 1, sampleRate); } const outBuffer = ctx.createBuffer(channels, outSamples, sampleRate); if (Math.abs(speed - 1) < 0.001) { const copyLength = Math.min(endSample - startSample, outSamples); if (copyLength > 0) { for (let c = 0; c < channels; c++) { outBuffer.copyToChannel( originalBuffer .getChannelData(c) .subarray(startSample, startSample + copyLength), c, ); } } return outBuffer; } // WSOLA uses windowing which causes fade-in at the start and fade-out at the end. // To avoid clicks at chunk boundaries, we render with 100ms of padding and trim it. const paddingSec = 0.1; const paddingOutSamples = Math.floor(sampleRate * paddingSec); const paddingInSamples = Math.floor(paddingOutSamples * speed); const workStartIn = Math.max(0, startSample - paddingInSamples); const workEndIn = Math.min(originalBuffer.length, endSample + paddingInSamples); const actualPaddingInStart = startSample - workStartIn; // We expect the output offset for the requested start to be roughly: const actualPaddingOutStart = Math.floor(actualPaddingInStart / speed); const windowSize = Math.floor(sampleRate * 0.04); const hopOut = Math.floor(windowSize * 0.5); const hopIn = Math.floor(hopOut * speed); const searchRange = Math.floor(sampleRate * 0.015); const workOutSamples = Math.floor((workEndIn - workStartIn) / speed) + windowSize * 2; const workOutBuffer = ctx.createBuffer(channels, workOutSamples, sampleRate); const inDataByChannel = Array.from({ length: channels }, (_, c) => originalBuffer.getChannelData(c), ); const workOutDataByChannel = Array.from({ length: channels }, (_, c) => workOutBuffer.getChannelData(c), ); const window = new Float32Array(windowSize); for (let i = 0; i < windowSize; i++) { window[i] = 0.5 * (1 - Math.cos((2 * Math.PI * i) / (windowSize - 1))); } let inOffset = workStartIn; let outOffset = 0; // Initial window for (let i = 0; i < windowSize; i++) { if (inOffset + i < workEndIn && outOffset + i < workOutSamples) { for (let c = 0; c < channels; c++) { workOutDataByChannel[c][outOffset + i] += inDataByChannel[c][inOffset + i] * window[i]; } } } outOffset += hopOut; inOffset += hopIn; while (outOffset + windowSize < workOutSamples && inOffset + windowSize < workEndIn) { let bestOffset = inOffset; const minSearch = Math.max(workStartIn, inOffset - searchRange); const maxSearch = Math.min(workEndIn - windowSize, inOffset + searchRange); if (maxSearch > minSearch) { let maxCorr = -Infinity; let bestDelta = 0; for (let testOffset = minSearch; testOffset <= maxSearch; testOffset += 4) { let corr = 0; for (let i = 0; i < hopOut; i += 4) { if (outOffset + i < workOutSamples && testOffset + i < workEndIn) { for (let c = 0; c < channels; c++) { corr += workOutDataByChannel[c][outOffset + i] * inDataByChannel[c][testOffset + i]; } } } if (corr > maxCorr) { maxCorr = corr; bestDelta = testOffset - inOffset; } } bestOffset = inOffset + bestDelta; } for (let i = 0; i < windowSize; i++) { if (bestOffset + i < workEndIn && outOffset + i < workOutSamples) { for (let c = 0; c < channels; c++) { workOutDataByChannel[c][outOffset + i] += inDataByChannel[c][bestOffset + i] * window[i]; } } } outOffset += hopOut; inOffset += hopIn; } // Transfer the stable middle portion to the final buffer for (let c = 0; c < channels; c++) { const finalData = outBuffer.getChannelData(c); const tempData = workOutBuffer.getChannelData(c); for (let i = 0; i < outSamples; i++) { const srcIdx = actualPaddingOutStart + i; if (srcIdx < workOutSamples) { finalData[i] = tempData[srcIdx]; } } } return outBuffer; } // Create a WAV file header for the given audio parameters. private createWavHeader( sampleRate: number, numChannels: number, totalFrames: number, ): ArrayBuffer { const bytesPerSample = 2; // 16-bit PCM const dataSize = totalFrames * numChannels * bytesPerSample; const headerSize = 44; const header = new ArrayBuffer(headerSize); const view = new DataView(header); const writeString = (offset: number, str: string) => { for (let i = 0; i < str.length; i++) { view.setUint8(offset + i, str.charCodeAt(i)); } }; writeString(0, "RIFF"); view.setUint32(4, headerSize - 8 + dataSize, true); writeString(8, "WAVE"); writeString(12, "fmt "); view.setUint32(16, 16, true); view.setUint16(20, 1, true); // PCM format view.setUint16(22, numChannels, true); view.setUint32(24, sampleRate, true); view.setUint32(28, sampleRate * numChannels * bytesPerSample, true); view.setUint16(32, numChannels * bytesPerSample, true); view.setUint16(34, bytesPerSample * 8, true); writeString(36, "data"); view.setUint32(40, dataSize, true); return header; } // Convert an AudioBuffer to chunked 16-bit PCM ArrayBuffers. // Returns small (~256KB) pieces instead of one massive allocation. private audioBufferToPcmParts(buffer: AudioBuffer): ArrayBuffer[] { const PCM_CHUNK_FRAMES = 65536; const numChannels = buffer.numberOfChannels; const numFrames = buffer.length; const bytesPerSample = 2; const parts: ArrayBuffer[] = []; const channels: Float32Array[] = []; for (let ch = 0; ch < numChannels; ch++) { channels.push(buffer.getChannelData(ch)); } for (let frameOffset = 0; frameOffset < numFrames; frameOffset += PCM_CHUNK_FRAMES) { const chunkFrames = Math.min(PCM_CHUNK_FRAMES, numFrames - frameOffset); const chunkBuffer = new ArrayBuffer(chunkFrames * numChannels * bytesPerSample); const view = new DataView(chunkBuffer); let byteOffset = 0; for (let i = 0; i < chunkFrames; i++) { for (let ch = 0; ch < numChannels; ch++) { const sample = Math.max(-1, Math.min(1, channels[ch][frameOffset + i])); view.setInt16(byteOffset, sample < 0 ? sample * 0x8000 : sample * 0x7fff, true); byteOffset += 2; } } parts.push(chunkBuffer); } return parts; } // Loads a sidecar audio file into a WebDemuxer for direct transcoding (avoiding real-time rendering). private async loadAudioFileDemuxer(audioPath: string): Promise { try { const source = await resolveMediaElementSource(audioPath); try { const wasmUrl = new URL("./wasm/web-demuxer.wasm", window.location.href).href; const demuxer = new WebDemuxer({ wasmFilePath: wasmUrl }); await demuxer.load(source.src); return demuxer; } finally { source.revoke(); } } catch (error) { console.warn("[AudioProcessor] Failed to create demuxer for sidecar audio:", error); return null; } } private cloneWithTimestamp(src: AudioData, newTimestamp: number): AudioData { const isPlanar = src.format?.includes("planar") ?? false; const numPlanes = isPlanar ? src.numberOfChannels : 1; let totalSize = 0; for (let planeIndex = 0; planeIndex < numPlanes; planeIndex++) { totalSize += src.allocationSize({ planeIndex }); } const buffer = new ArrayBuffer(totalSize); let offset = 0; for (let planeIndex = 0; planeIndex < numPlanes; planeIndex++) { const planeSize = src.allocationSize({ planeIndex }); src.copyTo(new Uint8Array(buffer, offset, planeSize), { planeIndex }); offset += planeSize; } return new AudioData({ format: src.format!, sampleRate: src.sampleRate, numberOfFrames: src.numberOfFrames, numberOfChannels: src.numberOfChannels, timestamp: newTimestamp, data: buffer, }); } private cloneEncodedAudioChunkWithTimestamp( src: EncodedAudioChunk, newTimestamp: number, ): EncodedAudioChunk { const data = new Uint8Array(src.byteLength); src.copyTo(data); return new EncodedAudioChunk({ type: src.type, timestamp: newTimestamp, duration: src.duration ?? undefined, data, }); } private isInTrimRegion(timestampMs: number, trims: TrimLikeRegion[]) { return trims.some((trim) => timestampMs >= trim.startMs && timestampMs < trim.endMs); } private computeTrimOffset(timestampMs: number, trims: TrimLikeRegion[]) { let offset = 0; for (const trim of trims) { if (trim.endMs <= timestampMs) { offset += trim.endMs - trim.startMs; } } return offset; } cancel() { this.cancelled = true; } }