Merge pull request #557 from webadderallorg/fix/voice-audio-continuity

fix(export): keep mic sidecar in native audio mix

src/lib/exporter/modernVideoExporter.nativeStaticLayout.test.ts

@@ -149,6 +149,21 @@ describe("ModernVideoExporter native static-layout eligibility", () => {
 		});
 	});
 
+	it("mixes companion sidecar audio when the source MP4 also has an audio track", () => {
+		const videoPath = "C:\\recordly\\recording.mp4";
+		const micPath = "C:\\recordly\\recording.mic.wav";
+		const exporter = createExporter({
+			videoUrl: `file:///${videoPath.replace(/\\/g, "/")}`,
+			sourceAudioFallbackPaths: [micPath],
+		});
+
+		expect(exporter.buildNativeAudioPlan(videoInfo)).toMatchObject({
+			audioMode: "edited-track",
+			strategy: "offline-render-fallback",
+			sourceAudioFallbackPaths: [expect.stringMatching(/recording\.mp4$/), micPath],
+		});
+	});
+
 	it("keeps timed companion audio on the offline render path", () => {
 		const audioPath = "C:\\recordly\\recording.system.wav";
 		const speedRegions: SpeedRegion[] = [
@@ -167,9 +182,10 @@ describe("ModernVideoExporter native static-layout eligibility", () => {
 				audioCodec: undefined,
 				audioSampleRate: undefined,
 			}),
-		).toEqual({
+		).toMatchObject({
 			audioMode: "edited-track",
 			strategy: "offline-render-fallback",
+			sourceAudioFallbackPaths: [audioPath],
 		});
 	});
 

src/lib/exporter/modernVideoExporter.ts

@@ -78,6 +78,7 @@ import {
 import { VideoMuxer } from "./muxer";
 import { roundNativeStaticLayoutContentSize } from "./nativeStaticLayoutGeometry";
 import { buildNativeStaticLayoutCursorTelemetry } from "./nativeStaticLayoutTelemetry";
+import { resolveSourceAudioFallbackPaths } from "./sourceAudioFallback";
 import { type DecodedVideoInfo, StreamingVideoDecoder } from "./streamingDecoder";
 import type {
 	ExportConfig,
@@ -164,6 +165,7 @@ type NativeAudioPlan =
 	| {
 			audioMode: "edited-track";
 			strategy: "offline-render-fallback";
+			sourceAudioFallbackPaths?: string[];
 	  }
 	| {
 			audioMode: "edited-track";
@@ -1224,6 +1226,26 @@ export class ModernVideoExporter {
 		return buildNativeStaticLayoutTimelineSegments(sourceSegments);
 	}
 
+	private getNativeAudioFallbackPaths(videoInfo: DecodedVideoInfo): string[] {
+		const sourceAudioFallbackPaths = (this.config.sourceAudioFallbackPaths ?? []).filter(
+			(audioPath) => typeof audioPath === "string" && audioPath.trim().length > 0,
+		);
+		const localVideoSourcePath = this.getNativeVideoSourcePath();
+		if (!videoInfo.hasAudio || !localVideoSourcePath) {
+			return sourceAudioFallbackPaths;
+		}
+
+		const { externalAudioPaths } = resolveSourceAudioFallbackPaths(
+			localVideoSourcePath,
+			sourceAudioFallbackPaths,
+		);
+		if (externalAudioPaths.length === 0) {
+			return sourceAudioFallbackPaths;
+		}
+
+		return [localVideoSourcePath, ...externalAudioPaths];
+	}
+
 	private shouldUseNativeStaticLayoutTimelineMap(
 		videoInfo: DecodedVideoInfo,
 		effectiveDurationSec: number,
@@ -1243,9 +1265,7 @@ export class ModernVideoExporter {
 	private buildNativeAudioPlan(videoInfo: DecodedVideoInfo): NativeAudioPlan {
 		const speedRegions = this.config.speedRegions ?? [];
 		const audioRegions = this.config.audioRegions ?? [];
-		const sourceAudioFallbackPaths = (this.config.sourceAudioFallbackPaths ?? []).filter(
-			(audioPath) => typeof audioPath === "string" && audioPath.trim().length > 0,
-		);
+		const sourceAudioFallbackPaths = this.getNativeAudioFallbackPaths(videoInfo);
 		const hasTimedSourceAudioFallback = sourceAudioFallbackPaths.some(
 			(audioPath) =>
 				(this.config.sourceAudioFallbackStartDelayMsByPath?.[audioPath] ?? 0) > 0,
@@ -1338,6 +1358,7 @@ export class ModernVideoExporter {
 			return {
 				audioMode: "edited-track",
 				strategy: "offline-render-fallback",
+				sourceAudioFallbackPaths,
 			};
 		}
 
@@ -1345,6 +1366,7 @@ export class ModernVideoExporter {
 			return {
 				audioMode: "edited-track",
 				strategy: "offline-render-fallback",
+				sourceAudioFallbackPaths,
 			};
 		}
 
@@ -1892,6 +1914,7 @@ export class ModernVideoExporter {
 	private async renderEditedAudioForNativeMux(
 		description: string,
 		onProgress: (progress: number) => void,
+		sourceAudioFallbackPaths = this.config.sourceAudioFallbackPaths,
 	) {
 		this.audioProcessor = new AudioProcessor();
 		this.audioProcessor.setOnProgress(onProgress);
@@ -1902,7 +1925,7 @@ export class ModernVideoExporter {
 					this.config.trimRegions,
 					this.config.speedRegions,
 					this.config.audioRegions,
-					this.config.sourceAudioFallbackPaths,
+					sourceAudioFallbackPaths,
 					this.config.sourceAudioFallbackStartDelayMsByPath,
 					this.config.sourceAudioTrackSettings,
 					this.config.clipRegions,
@@ -1950,6 +1973,7 @@ export class ModernVideoExporter {
 					"Native static-layout edited audio rendering",
 					(progress) =>
 						this.reportProgress(0, totalFrames, "preparing", undefined, progress),
+					audioPlan.sourceAudioFallbackPaths,
 				);
 
 				return {
@@ -2738,6 +2762,7 @@ export class ModernVideoExporter {
 			const renderedAudio = await this.renderEditedAudioForNativeMux(
 				`${NATIVE_EXPORT_ENGINE_NAME} edited audio rendering`,
 				(progress) => this.reportFinalizingProgress(this.processedFrameCount, 99, progress),
+				audioPlan.sourceAudioFallbackPaths,
 			);
 			editedAudioBuffer = renderedAudio.editedAudioData;
 			editedAudioMimeType = renderedAudio.editedAudioMimeType;
@@ -2835,6 +2860,7 @@ export class ModernVideoExporter {
 			const renderedAudio = await this.renderEditedAudioForNativeMux(
 				"FFmpeg edited audio rendering",
 				(progress) => this.reportFinalizingProgress(this.processedFrameCount, 99, progress),
+				audioPlan.sourceAudioFallbackPaths,
 			);
 			editedAudioBuffer = renderedAudio.editedAudioData;
 			editedAudioMimeType = renderedAudio.editedAudioMimeType;