diff --git a/src/components/video-editor/VideoEditor.tsx b/src/components/video-editor/VideoEditor.tsx index d55c64b6..7342ed93 100644 --- a/src/components/video-editor/VideoEditor.tsx +++ b/src/components/video-editor/VideoEditor.tsx @@ -158,8 +158,8 @@ import { extendAutoFullTrackClip, type FigureData, getClipSourceEndMs, - type PlaybackSpeed, type Padding, + type PlaybackSpeed, type SpeedRegion, type TrimRegion, type WebcamOverlaySettings, @@ -752,7 +752,9 @@ export default function VideoEditor() { } context.imageSmoothingEnabled = true; context.imageSmoothingQuality = "high"; - const editorBgHsl = getComputedStyle(document.documentElement).getPropertyValue("--editor-bg").trim(); + const editorBgHsl = getComputedStyle(document.documentElement) + .getPropertyValue("--editor-bg") + .trim(); context.fillStyle = editorBgHsl ? `hsl(${editorBgHsl})` : "#111113"; context.fillRect(0, 0, targetWidth, targetHeight); @@ -788,7 +790,9 @@ export default function VideoEditor() { padding, cropRegion, webcam, - webcamUrl: resolvedWebcamVideoUrl ?? (webcam.sourcePath ? toFileUrl(webcam.sourcePath) : null), + webcamUrl: + resolvedWebcamVideoUrl ?? + (webcam.sourcePath ? toFileUrl(webcam.sourcePath) : null), videoWidth: previewVideo.videoWidth, videoHeight: previewVideo.videoHeight, annotationRegions, @@ -1205,10 +1209,7 @@ export default function VideoEditor() { () => videoSourcePath ?? (videoPath ? fromFileUrl(videoPath) : null), [videoPath, videoSourcePath], ); - const { - hasEmbeddedSourceAudio, - externalAudioPaths: previewSourceAudioFallbackPaths, - } = useMemo( + const { hasEmbeddedSourceAudio, externalAudioPaths: previewSourceAudioFallbackPaths } = useMemo( () => resolveSourceAudioFallbackPaths(currentSourcePath, sourceAudioFallbackPaths), [currentSourcePath, sourceAudioFallbackPaths], ); @@ -2160,7 +2161,7 @@ export default function VideoEditor() { currentSourcePath, currentPersistedEditorState, lastSavedSnapshot?.projectId ?? null, - ); + ); const fileNameBase = currentSourcePath @@ -2275,7 +2276,7 @@ export default function VideoEditor() { currentSourcePath, currentPersistedEditorState, lastSavedSnapshot?.projectId ?? null, - ); + ); const thumbnailDataUrl = await captureProjectThumbnail(); const result = await window.electronAPI.saveProjectFileNamed( projectData, @@ -2975,7 +2976,9 @@ export default function VideoEditor() { regions.filter( (region) => !removedSegments.some( - (segment) => region.startMs < segment.endMs && region.endMs > segment.startMs, + (segment) => + region.startMs < segment.endMs && + region.endMs > segment.startMs, ), ); setZoomRegions((prev) => removeTrimmedRegions(prev)); @@ -3510,20 +3513,39 @@ export default function VideoEditor() { sourceAudioElementResourcesRef.current.set(audioPath, audioPath); void (async () => { - const resolved = await resolveMediaElementSource(audioPath); - const latestAudio = existing.get(audioPath); + try { + const resolved = await resolveMediaElementSource(audioPath); + const latestAudio = existing.get(audioPath); - if ( - cancelled || - latestAudio !== audio || - sourceAudioElementResourcesRef.current.get(audioPath) !== audioPath - ) { - resolved.revoke(); - return; + if ( + cancelled || + latestAudio !== audio || + sourceAudioElementResourcesRef.current.get(audioPath) !== audioPath + ) { + resolved.revoke(); + return; + } + + sourceAudioElementRevokersRef.current.set(audioPath, resolved.revoke); + latestAudio.src = resolved.src; + } catch (error) { + if (cancelled) { + return; + } + + sourceAudioElementRevokersRef.current.get(audioPath)?.(); + sourceAudioElementRevokersRef.current.delete(audioPath); + sourceAudioElementResourcesRef.current.delete(audioPath); + const latestAudio = existing.get(audioPath); + if (latestAudio === audio) { + latestAudio.pause(); + latestAudio.src = ""; + } + toast.warning( + `Could not load companion audio source: ${summarizeErrorMessage(getErrorMessage(error))}`, + { id: SOURCE_AUDIO_FALLBACK_TOAST_ID, duration: 10000 }, + ); } - - sourceAudioElementRevokersRef.current.set(audioPath, resolved.revoke); - latestAudio.src = resolved.src; })(); } @@ -3785,7 +3807,9 @@ export default function VideoEditor() { videoPadding: padding, cropRegion, webcam, - webcamUrl: resolvedWebcamVideoUrl ?? (webcam.sourcePath ? toFileUrl(webcam.sourcePath) : null), + webcamUrl: + resolvedWebcamVideoUrl ?? + (webcam.sourcePath ? toFileUrl(webcam.sourcePath) : null), annotationRegions, autoCaptions, autoCaptionSettings, @@ -3954,7 +3978,9 @@ export default function VideoEditor() { padding, cropRegion, webcam, - webcamUrl: resolvedWebcamVideoUrl ?? (webcam.sourcePath ? toFileUrl(webcam.sourcePath) : null), + webcamUrl: + resolvedWebcamVideoUrl ?? + (webcam.sourcePath ? toFileUrl(webcam.sourcePath) : null), annotationRegions, autoCaptions, autoCaptionSettings, @@ -4708,7 +4734,10 @@ export default function VideoEditor() {

{isRenderingAudio ? (

- {t("editor.export.processingAudioEdits", "Processing audio with speed/overlay edits")} + {t( + "editor.export.processingAudioEdits", + "Processing audio with speed/overlay edits", + )}

) : exportRenderSpeedLabel ? (

diff --git a/src/lib/exporter/audioEncoder.test.ts b/src/lib/exporter/audioEncoder.test.ts index 2b67ddaa..a39e9358 100644 --- a/src/lib/exporter/audioEncoder.test.ts +++ b/src/lib/exporter/audioEncoder.test.ts @@ -2,14 +2,38 @@ import { describe, expect, it, vi } from "vitest"; import { AudioProcessor } from "./audioEncoder"; +type OfflineRenderTestHarness = AudioProcessor & { + decodeAudioFromUrl(url: string): Promise; + getMediaDurationSec(url: string): Promise; + loadAudioFileDemuxer(audioPath: string): Promise; + prepareOfflineRender( + videoUrl: string, + trimRegions: never[], + speedRegions: never[], + audioRegions: never[], + sourceAudioFallbackPaths: string[], + ): Promise<{ + mainBuffer: AudioBuffer | null; + companionEntries: Array<{ buffer: AudioBuffer; startDelaySec: number }>; + }>; + renderAndMuxOfflineAudio( + videoUrl: string, + trimRegions: never[], + speedRegions: never[], + audioRegions: never[], + sourceAudioFallbackPaths: string[], + muxer: unknown, + ): Promise; +}; + describe("AudioProcessor offline render preparation", () => { it("keeps embedded source audio separate from external companion sidecars", async () => { - const processor = new AudioProcessor(); + const processor = new AudioProcessor() as unknown as OfflineRenderTestHarness; const mainBuffer = { duration: 10, numberOfChannels: 2 } as AudioBuffer; const micBuffer = { duration: 9.5, numberOfChannels: 1 } as AudioBuffer; const decodeAudioFromUrl = vi - .spyOn(processor as never, "decodeAudioFromUrl") + .spyOn(processor, "decodeAudioFromUrl") .mockImplementation(async (url: string) => { if (url === "file:///tmp/recording.mp4") { return mainBuffer; @@ -19,9 +43,9 @@ describe("AudioProcessor offline render preparation", () => { } return null; }); - vi.spyOn(processor as never, "getMediaDurationSec").mockResolvedValue(10); + vi.spyOn(processor, "getMediaDurationSec").mockResolvedValue(10); - const prepared = await (processor as never).prepareOfflineRender( + const prepared = await processor.prepareOfflineRender( "file:///tmp/recording.mp4", [], [], @@ -36,4 +60,26 @@ describe("AudioProcessor offline render preparation", () => { expect(decodeAudioFromUrl).toHaveBeenCalledWith("/tmp/recording.mic.wav"); expect(decodeAudioFromUrl).not.toHaveBeenCalledWith("/tmp/recording.mp4"); }); -}); \ No newline at end of file + + it("does not treat a single embedded fallback path as an external sidecar", async () => { + const processor = new AudioProcessor() as unknown as OfflineRenderTestHarness; + const loadAudioFileDemuxer = vi.spyOn(processor, "loadAudioFileDemuxer"); + const renderAndMuxOfflineAudio = vi + .spyOn(processor, "renderAndMuxOfflineAudio") + .mockResolvedValue(); + + await processor.process( + null, + {} as never, + "file:///tmp/recording.mp4", + [], + [], + undefined, + [], + ["/tmp/recording.mp4"], + ); + + expect(loadAudioFileDemuxer).not.toHaveBeenCalled(); + expect(renderAndMuxOfflineAudio).not.toHaveBeenCalled(); + }); +}); diff --git a/src/lib/exporter/audioEncoder.ts b/src/lib/exporter/audioEncoder.ts index 5b8c83bf..150fbfea 100644 --- a/src/lib/exporter/audioEncoder.ts +++ b/src/lib/exporter/audioEncoder.ts @@ -5,9 +5,7 @@ import type { SpeedRegion, TrimRegion, } from "@/components/video-editor/types"; -import { - estimateCompanionAudioStartDelaySeconds, -} from "@/lib/mediaTiming"; +import { estimateCompanionAudioStartDelaySeconds } from "@/lib/mediaTiming"; import { resolveMediaElementSource } from "./localMediaSource"; import type { VideoMuxer } from "./muxer"; import { resolveSourceAudioFallbackPaths } from "./sourceAudioFallback"; @@ -37,20 +35,20 @@ interface PreparedOfflineRender { } export async function isAacAudioEncodingSupported( - sampleRate = 48_000, - numberOfChannels = 2, + sampleRate = 48_000, + numberOfChannels = 2, ): Promise { - try { - const support = await AudioEncoder.isConfigSupported({ - codec: MP4_AUDIO_CODEC, - sampleRate, - numberOfChannels, - bitrate: AUDIO_BITRATE, - }); - return support.supported === true; - } catch { - return false; - } + try { + const support = await AudioEncoder.isConfigSupported({ + codec: MP4_AUDIO_CODEC, + sampleRate, + numberOfChannels, + bitrate: AUDIO_BITRATE, + }); + return support.supported === true; + } catch { + return false; + } } type TrimLikeRegion = TrimRegion | ClipRegion; @@ -162,12 +160,19 @@ export class AudioProcessor { (audioPath) => typeof audioPath === "string" && audioPath.trim().length > 0, ) : []; + const { hasEmbeddedSourceAudio, externalAudioPaths } = resolveSourceAudioFallbackPaths( + videoUrl, + sortedSourceAudioFallbackPaths, + ); + const needsSourceAudioMixing = + externalAudioPaths.length > 1 || + (hasEmbeddedSourceAudio && externalAudioPaths.length > 0); // When speed edits, audio regions, or multiple audio sources need mixing, use offline AudioContext pipeline. if ( sortedSpeedRegions.length > 0 || sortedAudioRegions.length > 0 || - sortedSourceAudioFallbackPaths.length > 1 + needsSourceAudioMixing ) { await this.renderAndMuxOfflineAudio( videoUrl, @@ -181,10 +186,8 @@ export class AudioProcessor { } // Single sidecar audio with no speed/audio edits: demux directly (skips slow real-time rendering). - if (sortedSourceAudioFallbackPaths.length === 1) { - const sidecarDemuxer = await this.loadAudioFileDemuxer( - sortedSourceAudioFallbackPaths[0], - ); + if (!hasEmbeddedSourceAudio && externalAudioPaths.length === 1) { + const sidecarDemuxer = await this.loadAudioFileDemuxer(externalAudioPaths[0]); if (sidecarDemuxer) { try { await this.processTrimOnlyAudio(sidecarDemuxer, muxer, sortedTrims); @@ -206,7 +209,7 @@ export class AudioProcessor { sortedTrims, [], [], - sortedSourceAudioFallbackPaths, + externalAudioPaths, muxer, ); return; @@ -564,8 +567,7 @@ export class AudioProcessor { const buffer = await this.decodeAudioFromUrl(audioPath); if (!buffer) continue; - const refDuration = - mainBuffer?.duration ?? (await this.getMediaDurationSec(videoUrl)); + const refDuration = mainBuffer?.duration ?? (await this.getMediaDurationSec(videoUrl)); companionEntries.push({ buffer, startDelaySec: estimateCompanionAudioStartDelaySeconds( @@ -661,10 +663,7 @@ export class AudioProcessor { pendingMuxing = pendingMuxing .then(async () => { if (this.cancelled) return; - await muxer.addAudioChunk( - chunk, - !wroteFirstChunk ? meta : undefined, - ); + await muxer.addAudioChunk(chunk, !wroteFirstChunk ? meta : undefined); wroteFirstChunk = true; }) .catch((error) => { @@ -708,27 +707,17 @@ export class AudioProcessor { // Render timeline to a WAV blob for the native/FFmpeg export path. // Processes in chunks to avoid holding the entire output in memory. - private async renderToWavBlobChunked( - prepared: PreparedOfflineRender, - ): Promise { + private async renderToWavBlobChunked(prepared: PreparedOfflineRender): Promise { const totalOutputSec = Math.max(prepared.outputDurationMs / 1000, 0.01); const totalFrames = Math.ceil(totalOutputSec * OFFLINE_AUDIO_SAMPLE_RATE); const numChannels = prepared.numChannels; - const header = this.createWavHeader( - OFFLINE_AUDIO_SAMPLE_RATE, - numChannels, - totalFrames, - ); + const header = this.createWavHeader(OFFLINE_AUDIO_SAMPLE_RATE, numChannels, totalFrames); const pcmParts: ArrayBuffer[] = [header]; - await this.renderChunked( - prepared, - totalOutputSec, - async (rendered) => { - pcmParts.push(...this.audioBufferToPcmParts(rendered)); - }, - ); + await this.renderChunked(prepared, totalOutputSec, async (rendered) => { + pcmParts.push(...this.audioBufferToPcmParts(rendered)); + }); return new Blob(pcmParts, { type: "audio/wav" }); } @@ -749,10 +738,7 @@ export class AudioProcessor { const chunkCount = Math.ceil(totalOutputSec / OFFLINE_CHUNK_DURATION_SEC); for (let i = 0; i < chunkCount && !this.cancelled; i++) { - const chunkSec = Math.min( - OFFLINE_CHUNK_DURATION_SEC, - totalOutputSec - outputOffsetSec, - ); + const chunkSec = Math.min(OFFLINE_CHUNK_DURATION_SEC, totalOutputSec - outputOffsetSec); const chunkFrames = Math.ceil(chunkSec * OFFLINE_AUDIO_SAMPLE_RATE); const offlineCtx = new OfflineAudioContext( @@ -835,10 +821,7 @@ export class AudioProcessor { localEndSec = chunkDurationSec; } - const duration = Math.min( - localEndSec - localStartSec, - buffer.duration - bufferOffsetSec, - ); + const duration = Math.min(localEndSec - localStartSec, buffer.duration - bufferOffsetSec); if (duration <= 0.001) return; const gainNode = ctx.createGain(); @@ -871,10 +854,7 @@ export class AudioProcessor { const planarData = new Float32Array(frameCount * numChannels); for (let ch = 0; ch < numChannels; ch++) { const channelData = buffer.getChannelData(ch); - planarData.set( - channelData.subarray(offset, offset + frameCount), - ch * frameCount, - ); + planarData.set(channelData.subarray(offset, offset + frameCount), ch * frameCount); } const audioData = new AudioData({ @@ -882,19 +862,14 @@ export class AudioProcessor { sampleRate, numberOfFrames: frameCount, numberOfChannels: numChannels, - timestamp: Math.round( - (offset / sampleRate + timestampOffsetSec) * 1_000_000, - ), + timestamp: Math.round((offset / sampleRate + timestampOffsetSec) * 1_000_000), data: planarData, }); encoder.encode(audioData); audioData.close(); - while ( - encoder.encodeQueueSize >= ENCODE_BACKPRESSURE_LIMIT && - !this.cancelled - ) { + while (encoder.encodeQueueSize >= ENCODE_BACKPRESSURE_LIMIT && !this.cancelled) { await new Promise((r) => setTimeout(r, 1)); } } @@ -931,18 +906,13 @@ export class AudioProcessor { type: blob.type || "video/mp4", }); - const wasmUrl = new URL( - "./wasm/web-demuxer.wasm", - window.location.href, - ).href; + const wasmUrl = new URL("./wasm/web-demuxer.wasm", window.location.href).href; demuxer = new WebDemuxer({ wasmFilePath: wasmUrl }); await demuxer.load(file); let audioConfig: AudioDecoderConfig; try { - audioConfig = (await demuxer.getDecoderConfig( - "audio", - )) as AudioDecoderConfig; + audioConfig = (await demuxer.getDecoderConfig("audio")) as AudioDecoderConfig; } catch { return null; // No audio track } @@ -951,10 +921,7 @@ export class AudioProcessor { const numChannels = Math.min(audioConfig.numberOfChannels || 2, 2); // Accumulate decoded PCM per channel - const channelChunks: Float32Array[][] = Array.from( - { length: numChannels }, - () => [], - ); + const channelChunks: Float32Array[][] = Array.from({ length: numChannels }, () => []); let totalFrames = 0; let decodeError: Error | null = null; @@ -962,10 +929,7 @@ export class AudioProcessor { output: (data: AudioData) => { try { const frames = data.numberOfFrames; - const dataChannels = Math.min( - data.numberOfChannels, - numChannels, - ); + const dataChannels = Math.min(data.numberOfChannels, numChannels); const format = data.format; if (format?.includes("planar")) { @@ -975,9 +939,7 @@ export class AudioProcessor { }); const bytes = new ArrayBuffer(size); data.copyTo(bytes, { planeIndex: ch }); - channelChunks[ch].push( - this.rawToFloat32(bytes, format, frames), - ); + channelChunks[ch].push(this.rawToFloat32(bytes, format, frames)); } } else if (format) { // Interleaved format — deinterleave into per-channel arrays. @@ -995,8 +957,7 @@ export class AudioProcessor { for (let ch = 0; ch < dataChannels; ch++) { const chData = new Float32Array(frames); for (let i = 0; i < frames; i++) { - chData[i] = - interleaved[i * srcChannels + ch]; + chData[i] = interleaved[i * srcChannels + ch]; } channelChunks[ch].push(chData); } @@ -1013,18 +974,14 @@ export class AudioProcessor { } }, error: (err: DOMException) => { - decodeError = new Error( - `Streaming audio decode error: ${err.message}`, - ); + decodeError = new Error(`Streaming audio decode error: ${err.message}`); }, }); decoder.configure(audioConfig); const audioStream = demuxer.read("audio"); - const reader = ( - audioStream as ReadableStream - ).getReader(); + const reader = (audioStream as ReadableStream).getReader(); try { while (!this.cancelled) { @@ -1034,10 +991,7 @@ export class AudioProcessor { decoder.decode(chunk); - while ( - decoder.decodeQueueSize > DECODE_BACKPRESSURE_LIMIT && - !this.cancelled - ) { + while (decoder.decodeQueueSize > DECODE_BACKPRESSURE_LIMIT && !this.cancelled) { if (decodeError) throw decodeError; await new Promise((r) => setTimeout(r, 1)); } @@ -1087,11 +1041,7 @@ export class AudioProcessor { } // Convert raw bytes from AudioData to Float32Array based on the sample format. - private rawToFloat32( - bytes: ArrayBuffer, - format: string, - sampleCount: number, - ): Float32Array { + private rawToFloat32(bytes: ArrayBuffer, format: string, sampleCount: number): Float32Array { if (format.startsWith("f32")) { return new Float32Array(bytes); } @@ -1124,10 +1074,7 @@ export class AudioProcessor { } // Bulk decode fallback: loads entire file into memory and uses decodeAudioData. - private async bulkDecodeFromUrl( - url: string, - sampleRate: number, - ): Promise { + private async bulkDecodeFromUrl(url: string, sampleRate: number): Promise { try { const source = await resolveMediaElementSource(url); try { @@ -1199,8 +1146,7 @@ export class AudioProcessor { boundaries.add(sourceDurationMs); for (const trim of trimRegions) { - if (trim.startMs >= 0 && trim.startMs <= sourceDurationMs) - boundaries.add(trim.startMs); + if (trim.startMs >= 0 && trim.startMs <= sourceDurationMs) boundaries.add(trim.startMs); if (trim.endMs >= 0 && trim.endMs <= sourceDurationMs) boundaries.add(trim.endMs); } for (const speed of speedRegions) { @@ -1236,10 +1182,7 @@ export class AudioProcessor { } // Map a source-timeline timestamp to the corresponding output-timeline timestamp. - private sourceTimeToOutputTime( - sourceMs: number, - slices: TimelineSlice[], - ): number { + private sourceTimeToOutputTime(sourceMs: number, slices: TimelineSlice[]): number { let outputMs = 0; for (const slice of slices) { @@ -1305,8 +1248,7 @@ export class AudioProcessor { // Calculate output position (global then chunk-local) let localOutputStartSec = outputOffsetSec + trimmedFromStartSec / slice.speed - chunkOutputStartSec; - let localOutputEndSec = - localOutputStartSec + effectiveSourceDurationSec / slice.speed; + let localOutputEndSec = localOutputStartSec + effectiveSourceDurationSec / slice.speed; // Skip if entirely outside chunk window if (localOutputEndSec <= 0 || localOutputStartSec >= chunkDurationSec) { @@ -1403,11 +1345,7 @@ export class AudioProcessor { for (let i = 0; i < chunkFrames; i++) { for (let ch = 0; ch < numChannels; ch++) { const sample = Math.max(-1, Math.min(1, channels[ch][frameOffset + i])); - view.setInt16( - byteOffset, - sample < 0 ? sample * 0x8000 : sample * 0x7fff, - true, - ); + view.setInt16(byteOffset, sample < 0 ? sample * 0x8000 : sample * 0x7fff, true); byteOffset += 2; } }