diff --git a/src/lib/exporter/modernVideoExporter.nativeStaticLayout.test.ts b/src/lib/exporter/modernVideoExporter.nativeStaticLayout.test.ts index 2d5d3c41..aee1b51d 100644 --- a/src/lib/exporter/modernVideoExporter.nativeStaticLayout.test.ts +++ b/src/lib/exporter/modernVideoExporter.nativeStaticLayout.test.ts @@ -149,6 +149,21 @@ describe("ModernVideoExporter native static-layout eligibility", () => { }); }); + it("mixes companion sidecar audio when the source MP4 also has an audio track", () => { + const videoPath = "C:\\recordly\\recording.mp4"; + const micPath = "C:\\recordly\\recording.mic.wav"; + const exporter = createExporter({ + videoUrl: `file:///${videoPath.replace(/\\/g, "/")}`, + sourceAudioFallbackPaths: [micPath], + }); + + expect(exporter.buildNativeAudioPlan(videoInfo)).toMatchObject({ + audioMode: "edited-track", + strategy: "offline-render-fallback", + sourceAudioFallbackPaths: [expect.stringMatching(/recording\.mp4$/), micPath], + }); + }); + it("keeps timed companion audio on the offline render path", () => { const audioPath = "C:\\recordly\\recording.system.wav"; const speedRegions: SpeedRegion[] = [ @@ -167,9 +182,10 @@ describe("ModernVideoExporter native static-layout eligibility", () => { audioCodec: undefined, audioSampleRate: undefined, }), - ).toEqual({ + ).toMatchObject({ audioMode: "edited-track", strategy: "offline-render-fallback", + sourceAudioFallbackPaths: [audioPath], }); }); diff --git a/src/lib/exporter/modernVideoExporter.ts b/src/lib/exporter/modernVideoExporter.ts index 3f9f39c5..cb192593 100644 --- a/src/lib/exporter/modernVideoExporter.ts +++ b/src/lib/exporter/modernVideoExporter.ts @@ -78,6 +78,7 @@ import { import { VideoMuxer } from "./muxer"; import { roundNativeStaticLayoutContentSize } from "./nativeStaticLayoutGeometry"; import { buildNativeStaticLayoutCursorTelemetry } from "./nativeStaticLayoutTelemetry"; +import { resolveSourceAudioFallbackPaths } from "./sourceAudioFallback"; import { type DecodedVideoInfo, StreamingVideoDecoder } from "./streamingDecoder"; import type { ExportConfig, @@ -164,6 +165,7 @@ type NativeAudioPlan = | { audioMode: "edited-track"; strategy: "offline-render-fallback"; + sourceAudioFallbackPaths?: string[]; } | { audioMode: "edited-track"; @@ -1224,6 +1226,26 @@ export class ModernVideoExporter { return buildNativeStaticLayoutTimelineSegments(sourceSegments); } + private getNativeAudioFallbackPaths(videoInfo: DecodedVideoInfo): string[] { + const sourceAudioFallbackPaths = (this.config.sourceAudioFallbackPaths ?? []).filter( + (audioPath) => typeof audioPath === "string" && audioPath.trim().length > 0, + ); + const localVideoSourcePath = this.getNativeVideoSourcePath(); + if (!videoInfo.hasAudio || !localVideoSourcePath) { + return sourceAudioFallbackPaths; + } + + const { externalAudioPaths } = resolveSourceAudioFallbackPaths( + localVideoSourcePath, + sourceAudioFallbackPaths, + ); + if (externalAudioPaths.length === 0) { + return sourceAudioFallbackPaths; + } + + return [localVideoSourcePath, ...externalAudioPaths]; + } + private shouldUseNativeStaticLayoutTimelineMap( videoInfo: DecodedVideoInfo, effectiveDurationSec: number, @@ -1243,9 +1265,7 @@ export class ModernVideoExporter { private buildNativeAudioPlan(videoInfo: DecodedVideoInfo): NativeAudioPlan { const speedRegions = this.config.speedRegions ?? []; const audioRegions = this.config.audioRegions ?? []; - const sourceAudioFallbackPaths = (this.config.sourceAudioFallbackPaths ?? []).filter( - (audioPath) => typeof audioPath === "string" && audioPath.trim().length > 0, - ); + const sourceAudioFallbackPaths = this.getNativeAudioFallbackPaths(videoInfo); const hasTimedSourceAudioFallback = sourceAudioFallbackPaths.some( (audioPath) => (this.config.sourceAudioFallbackStartDelayMsByPath?.[audioPath] ?? 0) > 0, @@ -1338,6 +1358,7 @@ export class ModernVideoExporter { return { audioMode: "edited-track", strategy: "offline-render-fallback", + sourceAudioFallbackPaths, }; } @@ -1345,6 +1366,7 @@ export class ModernVideoExporter { return { audioMode: "edited-track", strategy: "offline-render-fallback", + sourceAudioFallbackPaths, }; } @@ -1892,6 +1914,7 @@ export class ModernVideoExporter { private async renderEditedAudioForNativeMux( description: string, onProgress: (progress: number) => void, + sourceAudioFallbackPaths = this.config.sourceAudioFallbackPaths, ) { this.audioProcessor = new AudioProcessor(); this.audioProcessor.setOnProgress(onProgress); @@ -1902,7 +1925,7 @@ export class ModernVideoExporter { this.config.trimRegions, this.config.speedRegions, this.config.audioRegions, - this.config.sourceAudioFallbackPaths, + sourceAudioFallbackPaths, this.config.sourceAudioFallbackStartDelayMsByPath, this.config.sourceAudioTrackSettings, this.config.clipRegions, @@ -1950,6 +1973,7 @@ export class ModernVideoExporter { "Native static-layout edited audio rendering", (progress) => this.reportProgress(0, totalFrames, "preparing", undefined, progress), + audioPlan.sourceAudioFallbackPaths, ); return { @@ -2738,6 +2762,7 @@ export class ModernVideoExporter { const renderedAudio = await this.renderEditedAudioForNativeMux( `${NATIVE_EXPORT_ENGINE_NAME} edited audio rendering`, (progress) => this.reportFinalizingProgress(this.processedFrameCount, 99, progress), + audioPlan.sourceAudioFallbackPaths, ); editedAudioBuffer = renderedAudio.editedAudioData; editedAudioMimeType = renderedAudio.editedAudioMimeType; @@ -2835,6 +2860,7 @@ export class ModernVideoExporter { const renderedAudio = await this.renderEditedAudioForNativeMux( "FFmpeg edited audio rendering", (progress) => this.reportFinalizingProgress(this.processedFrameCount, 99, progress), + audioPlan.sourceAudioFallbackPaths, ); editedAudioBuffer = renderedAudio.editedAudioData; editedAudioMimeType = renderedAudio.editedAudioMimeType;