From d4fe5d72a1b5a4c945305510b86ae26372b7f796 Mon Sep 17 00:00:00 2001 From: Alan Trebugeais Date: Sat, 9 May 2026 18:51:44 +0200 Subject: [PATCH] audio encoding works with exporting. --- src/components/video-editor/VideoEditor.tsx | 3 +- .../video-editor/audio/useAudioPreviewSync.ts | 132 +++++++++++++++--- .../video-editor/audio/useVideoEditorAudio.ts | 14 +- src/lib/exporter/audioEncoder.ts | 77 +++++++--- src/lib/exporter/modernVideoExporter.ts | 4 + .../exporter/sourceTrackRoutingPolicy.test.ts | 41 ++++++ src/lib/exporter/sourceTrackRoutingPolicy.ts | 50 +++++++ src/lib/exporter/videoExporter.ts | 4 + 8 files changed, 274 insertions(+), 51 deletions(-) create mode 100644 src/lib/exporter/sourceTrackRoutingPolicy.test.ts create mode 100644 src/lib/exporter/sourceTrackRoutingPolicy.ts diff --git a/src/components/video-editor/VideoEditor.tsx b/src/components/video-editor/VideoEditor.tsx index fbc61a63..5ca4c864 100644 --- a/src/components/video-editor/VideoEditor.tsx +++ b/src/components/video-editor/VideoEditor.tsx @@ -4405,6 +4405,7 @@ export default function VideoEditor() { sourceAudioFallbackPaths: audio.sourceAudioFallbackPaths, sourceAudioFallbackStartDelayMsByPath: audio.sourceAudioFallbackStartDelayMsByPath, + sourceAudioTrackSettings: audio.activeSourceAudioTrackSettings, previewWidth, previewHeight, onProgress: (progress: ExportProgress) => { @@ -6064,7 +6065,7 @@ export default function VideoEditor() { } cursorSway={cursorSway} volume={ - audio.isCurrentClipMuted + audio.shouldMutePreviewVideo || audio.isCurrentClipMuted ? 0 : Math.max( 0, diff --git a/src/components/video-editor/audio/useAudioPreviewSync.ts b/src/components/video-editor/audio/useAudioPreviewSync.ts index 00b2b3b9..3d05c858 100644 --- a/src/components/video-editor/audio/useAudioPreviewSync.ts +++ b/src/components/video-editor/audio/useAudioPreviewSync.ts @@ -10,9 +10,6 @@ import type { AudioRegion, SpeedRegion } from "../types"; const SOURCE_AUDIO_PREVIEW_PLAYING_SEEK_DRIFT_SECONDS = 0.18; const SOURCE_AUDIO_PREVIEW_PAUSED_SEEK_DRIFT_SECONDS = 0.01; -const SOURCE_AUDIO_PREVIEW_RATE_TOLERANCE_SECONDS = 0.08; -const SOURCE_AUDIO_PREVIEW_RATE_CORRECTION_WINDOW_SECONDS = 8; -const SOURCE_AUDIO_PREVIEW_MAX_RATE_ADJUSTMENT = 0.015; interface UseAudioPreviewSyncParams { audioRegions: AudioRegion[]; @@ -47,10 +44,43 @@ export function useAudioPreviewSync({ const audioElementRevokersRef = useRef void>>(new Map()); const audioElementResourcesRef = useRef>(new Map()); const sourceAudioElementsRef = useRef>(new Map()); + const sourceAudioMediaNodesRef = useRef>(new Map()); + const sourceAudioGainNodesRef = useRef>(new Map()); const sourceAudioElementRevokersRef = useRef void>>(new Map()); const sourceAudioElementResourcesRef = useRef>(new Map()); + const sourceAudioContextRef = useRef(null); + const sourceAudioMasterGainRef = useRef(null); + const sourceAudioResumePromiseRef = useRef | null>(null); const lastSourceAudioSyncTimeRef = useRef(null); + const ensureSourceAudioContext = () => { + if (!sourceAudioContextRef.current) { + const context = new AudioContext({ latencyHint: "interactive" }); + const masterGain = context.createGain(); + masterGain.gain.value = 1; + masterGain.connect(context.destination); + sourceAudioContextRef.current = context; + sourceAudioMasterGainRef.current = masterGain; + } + return sourceAudioContextRef.current; + }; + + const ensureSourceAudioRunning = () => { + const context = ensureSourceAudioContext(); + if (context.state === "running") { + return Promise.resolve(); + } + if (!sourceAudioResumePromiseRef.current) { + sourceAudioResumePromiseRef.current = context + .resume() + .catch(() => undefined) + .finally(() => { + sourceAudioResumePromiseRef.current = null; + }); + } + return sourceAudioResumePromiseRef.current; + }; + useEffect(() => { let cancelled = false; const existing = audioElementsRef.current; @@ -117,6 +147,10 @@ export function useAudioPreviewSync({ if (!currentIds.has(id)) { audio.pause(); audio.src = ""; + sourceAudioMediaNodesRef.current.get(id)?.disconnect(); + sourceAudioMediaNodesRef.current.delete(id); + sourceAudioGainNodesRef.current.get(id)?.disconnect(); + sourceAudioGainNodesRef.current.delete(id); sourceAudioElementRevokersRef.current.get(id)?.(); sourceAudioElementRevokersRef.current.delete(id); sourceAudioElementResourcesRef.current.delete(id); @@ -129,10 +163,27 @@ export function useAudioPreviewSync({ if (!audio) { audio = new Audio(); audio.preload = "auto"; + audio.crossOrigin = "anonymous"; existing.set(audioPath, audio); } + audio.volume = 1; audio.dataset.sourceAudioPath = audioPath; + const context = ensureSourceAudioContext(); + const masterGain = sourceAudioMasterGainRef.current; + if (context && masterGain && !sourceAudioMediaNodesRef.current.has(audioPath)) { + try { + const mediaNode = context.createMediaElementSource(audio); + const trackGainNode = context.createGain(); + mediaNode.connect(trackGainNode); + trackGainNode.connect(masterGain); + sourceAudioMediaNodesRef.current.set(audioPath, mediaNode); + sourceAudioGainNodesRef.current.set(audioPath, trackGainNode); + } catch (error) { + onSourceFallbackLoadError(error); + } + } + if (sourceAudioElementResourcesRef.current.get(audioPath) !== audioPath) { audio.pause(); audio.src = ""; @@ -174,9 +225,16 @@ export function useAudioPreviewSync({ })(); } - audio.volume = isCurrentClipMuted + const trackGainNode = sourceAudioGainNodesRef.current.get(audioPath); + if (trackGainNode) { + trackGainNode.gain.value = Math.max(0, Math.min(2, getSourceTrackPreviewGain(audioPath))); + } + } + + if (sourceAudioMasterGainRef.current) { + sourceAudioMasterGainRef.current.gain.value = isCurrentClipMuted ? 0 - : Math.max(0, Math.min(1, previewVolume * getSourceTrackPreviewGain(audioPath))); + : Math.max(0, Math.min(1, previewVolume)); } if (previewSourceAudioFallbackPaths.length === 0) { @@ -210,12 +268,30 @@ export function useAudioPreviewSync({ audio.pause(); audio.src = ""; } + for (const node of sourceAudioMediaNodesRef.current.values()) { + node.disconnect(); + } + for (const node of sourceAudioGainNodesRef.current.values()) { + node.disconnect(); + } for (const revoke of sourceAudioElementRevokersRef.current.values()) { revoke(); } sourceAudioElementsRef.current.clear(); + sourceAudioMediaNodesRef.current.clear(); + sourceAudioGainNodesRef.current.clear(); sourceAudioElementRevokersRef.current.clear(); sourceAudioElementResourcesRef.current.clear(); + if (sourceAudioMasterGainRef.current) { + sourceAudioMasterGainRef.current.disconnect(); + sourceAudioMasterGainRef.current = null; + } + const context = sourceAudioContextRef.current; + sourceAudioContextRef.current = null; + sourceAudioResumePromiseRef.current = null; + if (context) { + void context.close(); + } lastSourceAudioSyncTimeRef.current = null; }; }, []); @@ -272,12 +348,18 @@ export function useAudioPreviewSync({ const driftThreshold = isPlaying ? SOURCE_AUDIO_PREVIEW_PLAYING_SEEK_DRIFT_SECONDS : SOURCE_AUDIO_PREVIEW_PAUSED_SEEK_DRIFT_SECONDS; + if (sourceAudioMasterGainRef.current) { + sourceAudioMasterGainRef.current.gain.value = isCurrentClipMuted + ? 0 + : Math.max(0, Math.min(1, previewVolume)); + } for (const audio of sourceAudioElementsRef.current.values()) { const sourceAudioPath = audio.dataset.sourceAudioPath ?? ""; - audio.volume = isCurrentClipMuted - ? 0 - : Math.max(0, Math.min(1, previewVolume * getSourceTrackPreviewGain(sourceAudioPath))); + const trackGainNode = sourceAudioGainNodesRef.current.get(sourceAudioPath); + if (trackGainNode) { + trackGainNode.gain.value = Math.max(0, Math.min(2, getSourceTrackPreviewGain(sourceAudioPath))); + } enablePitchPreservingPlayback(audio); const audioDuration = Number.isFinite(audio.duration) ? audio.duration : null; @@ -298,7 +380,11 @@ export function useAudioPreviewSync({ const beforeAudioStart = currentTime + 0.001 < startDelaySeconds; const targetTime = clampMediaTimeToDuration(currentTime - startDelaySeconds, audioDuration); - if (timelineJumped || Math.abs(audio.currentTime - targetTime) > driftThreshold) { + const shouldSeek = + timelineJumped || + (!isPlaying && Math.abs(audio.currentTime - targetTime) > driftThreshold) || + (isPlaying && Math.abs(audio.currentTime - targetTime) > 0.9); + if (shouldSeek) { try { audio.currentTime = targetTime; } catch { @@ -306,21 +392,18 @@ export function useAudioPreviewSync({ } } - const syncedPlaybackRate = getMediaSyncPlaybackRate({ - basePlaybackRate: targetPlaybackRate, - currentTime: audio.currentTime, - targetTime, - toleranceSeconds: SOURCE_AUDIO_PREVIEW_RATE_TOLERANCE_SECONDS, - correctionWindowSeconds: SOURCE_AUDIO_PREVIEW_RATE_CORRECTION_WINDOW_SECONDS, - maxAdjustment: SOURCE_AUDIO_PREVIEW_MAX_RATE_ADJUSTMENT, - }); + // KISS for companion source tracks: fixed playback rate avoids audible flutter/stutter + // from continuous micro-corrections on system audio. + const syncedPlaybackRate = targetPlaybackRate; if (Math.abs(audio.playbackRate - syncedPlaybackRate) > 0.001) { audio.playbackRate = syncedPlaybackRate; } const atEnd = audioDuration !== null && targetTime >= audioDuration; if (isPlaying && !beforeAudioStart && !atEnd) { - audio.play().catch(() => undefined); + void ensureSourceAudioRunning().then(() => { + audio.play().catch(() => undefined); + }); } else if (!audio.paused) { audio.pause(); } @@ -338,4 +421,17 @@ export function useAudioPreviewSync({ previewSourceAudioFallbackPaths, sourceAudioFallbackStartDelayMsByPath, ]); + + useEffect(() => { + if (!isPlaying || previewSourceAudioFallbackPaths.length === 0) { + return; + } + void ensureSourceAudioRunning().then(() => { + for (const audio of sourceAudioElementsRef.current.values()) { + if (audio.paused) { + audio.play().catch(() => undefined); + } + } + }); + }, [isPlaying, previewSourceAudioFallbackPaths]); } diff --git a/src/components/video-editor/audio/useVideoEditorAudio.ts b/src/components/video-editor/audio/useVideoEditorAudio.ts index d31f7d24..d992e871 100644 --- a/src/components/video-editor/audio/useVideoEditorAudio.ts +++ b/src/components/video-editor/audio/useVideoEditorAudio.ts @@ -1,5 +1,5 @@ import React, { useMemo } from "react"; -import { resolveSourceAudioFallbackPaths } from "@/lib/exporter/sourceAudioFallback"; +import { resolveSourceTrackRoutingPolicy } from "@/lib/exporter/sourceTrackRoutingPolicy"; import type { AudioRegion, ClipRegion, @@ -78,16 +78,12 @@ export function useVideoEditorAudio({ summarizeErrorMessage, }); - const { hasEmbeddedSourceAudio, externalAudioPaths: previewSourceAudioFallbackPaths } = useMemo( - () => resolveSourceAudioFallbackPaths(currentSourcePath, sourceAudioFallbackPaths), + const sourceTrackRoutingPolicy = useMemo( + () => resolveSourceTrackRoutingPolicy(currentSourcePath, sourceAudioFallbackPaths), [currentSourcePath, sourceAudioFallbackPaths], ); - const hasSystemCompanionPreviewTrack = previewSourceAudioFallbackPaths.some((audioPath) => - audioPath.toLowerCase().includes(".system."), - ); - const shouldMutePreviewVideo = - previewSourceAudioFallbackPaths.length > 0 && - (!hasEmbeddedSourceAudio || hasSystemCompanionPreviewTrack); + const previewSourceAudioFallbackPaths = sourceTrackRoutingPolicy.playbackPaths; + const shouldMutePreviewVideo = sourceTrackRoutingPolicy.muteEmbeddedPreview; const activeClipIdAtCurrentTime = useMemo( () => getActiveClipIdAtSourceTime(currentTime, clipRegions), diff --git a/src/lib/exporter/audioEncoder.ts b/src/lib/exporter/audioEncoder.ts index 647c9768..c6f014f8 100644 --- a/src/lib/exporter/audioEncoder.ts +++ b/src/lib/exporter/audioEncoder.ts @@ -3,12 +3,16 @@ import type { AudioRegion, ClipRegion, SpeedRegion, + SourceAudioTrackSettings, TrimRegion, } from "@/components/video-editor/types"; import { estimateCompanionAudioStartDelaySeconds } from "@/lib/mediaTiming"; import { resolveMediaElementSource } from "./localMediaSource"; import type { VideoMuxer } from "./muxer"; -import { resolveSourceAudioFallbackPaths } from "./sourceAudioFallback"; +import { + getSourceTrackIdFromPath, + resolveSourceTrackRoutingPolicy, +} from "./sourceTrackRoutingPolicy"; const AUDIO_BITRATE = 128_000; const DECODE_BACKPRESSURE_LIMIT = 20; @@ -26,8 +30,8 @@ interface TimelineSlice { } interface PreparedOfflineRender { - mainBuffer: AudioBuffer | null; - companionEntries: Array<{ buffer: AudioBuffer; startDelaySec: number }>; + mainBufferEntry: { buffer: AudioBuffer; gain: number } | null; + companionEntries: Array<{ buffer: AudioBuffer; startDelaySec: number; gain: number }>; regionEntries: Array<{ buffer: AudioBuffer; region: AudioRegion }>; slices: TimelineSlice[]; outputDurationMs: number; @@ -144,6 +148,7 @@ export class AudioProcessor { audioRegions?: AudioRegion[], sourceAudioFallbackPaths?: string[], sourceAudioFallbackStartDelayMsByPath?: Record, + sourceAudioTrackSettings?: SourceAudioTrackSettings, ): Promise { const sortedTrims = trimRegions ? [...trimRegions].sort((a, b) => a.startMs - b.startMs) @@ -161,16 +166,16 @@ export class AudioProcessor { (audioPath) => typeof audioPath === "string" && audioPath.trim().length > 0, ) : []; - const { hasEmbeddedSourceAudio, externalAudioPaths } = resolveSourceAudioFallbackPaths( + const routingPolicy = resolveSourceTrackRoutingPolicy( videoUrl, sortedSourceAudioFallbackPaths, ); - const hasTimedCompanionAudio = externalAudioPaths.some( + const hasTimedCompanionAudio = routingPolicy.playbackPaths.some( (audioPath) => (sourceAudioFallbackStartDelayMsByPath?.[audioPath] ?? 0) > 0, ); const needsSourceAudioMixing = - externalAudioPaths.length > 1 || - (hasEmbeddedSourceAudio && externalAudioPaths.length > 0) || + routingPolicy.playbackPaths.length > 1 || + (routingPolicy.hasEmbeddedSourceAudio && routingPolicy.playbackPaths.length > 0) || hasTimedCompanionAudio; // When speed edits, audio regions, or multiple audio sources need mixing, use offline AudioContext pipeline. @@ -186,14 +191,15 @@ export class AudioProcessor { sortedAudioRegions, sortedSourceAudioFallbackPaths, sourceAudioFallbackStartDelayMsByPath, + sourceAudioTrackSettings, muxer, ); return; } // Single sidecar audio with no speed/audio edits: demux directly (skips slow real-time rendering). - if (!hasEmbeddedSourceAudio && externalAudioPaths.length === 1) { - const sidecarDemuxer = await this.loadAudioFileDemuxer(externalAudioPaths[0]); + if (!routingPolicy.hasEmbeddedSourceAudio && routingPolicy.playbackPaths.length === 1) { + const sidecarDemuxer = await this.loadAudioFileDemuxer(routingPolicy.playbackPaths[0]); if (sidecarDemuxer) { try { await this.processTrimOnlyAudio(sidecarDemuxer, muxer, sortedTrims); @@ -215,8 +221,9 @@ export class AudioProcessor { sortedTrims, [], [], - externalAudioPaths, + routingPolicy.playbackPaths, sourceAudioFallbackStartDelayMsByPath, + sourceAudioTrackSettings, muxer, ); return; @@ -263,6 +270,7 @@ export class AudioProcessor { audioRegions?: AudioRegion[], sourceAudioFallbackPaths?: string[], sourceAudioFallbackStartDelayMsByPath?: Record, + sourceAudioTrackSettings?: SourceAudioTrackSettings, ): Promise { const sortedTrims = trimRegions ? [...trimRegions].sort((a, b) => a.startMs - b.startMs) @@ -288,6 +296,7 @@ export class AudioProcessor { sortedAudioRegions, sortedSourceAudioFallbackPaths, sourceAudioFallbackStartDelayMsByPath, + sourceAudioTrackSettings, ); return this.renderToWavBlobChunked(prepared); } @@ -563,6 +572,7 @@ export class AudioProcessor { audioRegions: AudioRegion[], sourceAudioFallbackPaths: string[], sourceAudioFallbackStartDelayMsByPath: Record | undefined, + sourceAudioTrackSettings: SourceAudioTrackSettings | undefined, muxer: VideoMuxer, ): Promise { const prepared = await this.prepareOfflineRender( @@ -572,6 +582,7 @@ export class AudioProcessor { audioRegions, sourceAudioFallbackPaths, sourceAudioFallbackStartDelayMsByPath, + sourceAudioTrackSettings, ); if (this.cancelled) return; await this.renderAndEncodeChunked(prepared, muxer); @@ -584,31 +595,45 @@ export class AudioProcessor { audioRegions: AudioRegion[], sourceAudioFallbackPaths: string[], sourceAudioFallbackStartDelayMsByPath?: Record, + sourceAudioTrackSettings?: SourceAudioTrackSettings, ): Promise { if (this.cancelled) throw new Error("Export cancelled"); this.onProgress?.(0); - const { externalAudioPaths } = resolveSourceAudioFallbackPaths( + const routingPolicy = resolveSourceTrackRoutingPolicy( videoUrl, sourceAudioFallbackPaths, ); // Decode embedded source audio separately from companion sidecars. - const mainBuffer = await this.decodeAudioFromUrl(videoUrl); + const mainBuffer = routingPolicy.includeEmbeddedInExport + ? await this.decodeAudioFromUrl(videoUrl) + : null; + const mainBufferGainSettings = + sourceAudioTrackSettings?.mixed ?? sourceAudioTrackSettings?.system ?? null; + const mainBufferGain = mainBufferGainSettings + ? Math.max(0, Math.min(2, mainBufferGainSettings.volume)) + : 1; + const mainBufferEntry = mainBuffer ? { buffer: mainBuffer, gain: mainBufferGain } : null; if (this.cancelled) throw new Error("Export cancelled"); // Decode companion / sidecar audio files - const companionEntries: Array<{ buffer: AudioBuffer; startDelaySec: number }> = []; + const companionEntries: Array<{ buffer: AudioBuffer; startDelaySec: number; gain: number }> = + []; const refDuration = mainBuffer?.duration ?? - (externalAudioPaths.length > 0 ? await this.getMediaDurationSec(videoUrl) : 0); - for (const audioPath of externalAudioPaths) { + (routingPolicy.playbackPaths.length > 0 ? await this.getMediaDurationSec(videoUrl) : 0); + for (const audioPath of routingPolicy.playbackPaths) { if (this.cancelled) throw new Error("Export cancelled"); const buffer = await this.decodeAudioFromUrl(audioPath); if (!buffer) continue; companionEntries.push({ buffer, + gain: Math.max( + 0, + Math.min(2, sourceAudioTrackSettings?.[getSourceTrackIdFromPath(audioPath)]?.volume ?? 1), + ), startDelaySec: estimateCompanionAudioStartDelaySeconds( refDuration, buffer.duration, @@ -629,15 +654,15 @@ export class AudioProcessor { this.onProgress?.(0.2); // Determine source duration for timeline calculation - const primaryBuffer = mainBuffer ?? companionEntries[0]?.buffer ?? null; + const primaryBuffer = mainBufferEntry?.buffer ?? companionEntries[0]?.buffer ?? null; if (!primaryBuffer && regionEntries.length === 0) { throw new Error("No decodable audio sources found"); } let sourceDurationSec: number; - if (mainBuffer) { - sourceDurationSec = mainBuffer.duration; - } else if (externalAudioPaths.length > 0 || regionEntries.length > 0) { + if (mainBufferEntry?.buffer) { + sourceDurationSec = mainBufferEntry.buffer.duration; + } else if (routingPolicy.playbackPaths.length > 0 || regionEntries.length > 0) { sourceDurationSec = await this.getMediaDurationSec(videoUrl); } else { sourceDurationSec = primaryBuffer?.duration ?? 0; @@ -661,7 +686,7 @@ export class AudioProcessor { const numChannels = Math.min(primaryBuffer?.numberOfChannels ?? 2, 2); return { - mainBuffer, + mainBufferEntry, companionEntries, regionEntries, slices, @@ -788,12 +813,13 @@ export class AudioProcessor { ); // Schedule main audio - if (prepared.mainBuffer) { + if (prepared.mainBufferEntry) { this.scheduleBufferThroughTimeline( offlineCtx, - prepared.mainBuffer, + prepared.mainBufferEntry.buffer, slices, 0, + prepared.mainBufferEntry.gain, outputOffsetSec, chunkSec, ); @@ -806,6 +832,7 @@ export class AudioProcessor { entry.buffer, slices, entry.startDelaySec, + entry.gain, outputOffsetSec, chunkSec, ); @@ -1265,6 +1292,7 @@ export class AudioProcessor { buffer: AudioBuffer, slices: TimelineSlice[], sourceStartDelaySec: number, + gain = 1, chunkOutputStartSec = 0, chunkDurationSec = Number.POSITIVE_INFINITY, ): void { @@ -1331,9 +1359,12 @@ export class AudioProcessor { } const source = ctx.createBufferSource(); + const gainNode = ctx.createGain(); + gainNode.gain.value = Math.max(0, Math.min(2, gain)); source.buffer = buffer; source.playbackRate.value = slice.speed; - source.connect(ctx.destination); + source.connect(gainNode); + gainNode.connect(ctx.destination); source.start(localOutputStartSec, effectiveBufferStartSec, effectiveSourceDurationSec); diff --git a/src/lib/exporter/modernVideoExporter.ts b/src/lib/exporter/modernVideoExporter.ts index ce8b3eae..424df4a0 100644 --- a/src/lib/exporter/modernVideoExporter.ts +++ b/src/lib/exporter/modernVideoExporter.ts @@ -8,6 +8,7 @@ import type { CursorTelemetryPoint, Padding, SpeedRegion, + SourceAudioTrackSettings, TrimRegion, WebcamOverlaySettings, ZoomMotionBlurTuning, @@ -137,6 +138,7 @@ interface VideoExporterConfig extends ExportConfig { audioRegions?: AudioRegion[]; sourceAudioFallbackPaths?: string[]; sourceAudioFallbackStartDelayMsByPath?: Record; + sourceAudioTrackSettings?: SourceAudioTrackSettings; previewWidth?: number; previewHeight?: number; onProgress?: (progress: ExportProgress) => void; @@ -752,6 +754,7 @@ export class ModernVideoExporter { this.config.audioRegions, this.config.sourceAudioFallbackPaths, this.config.sourceAudioFallbackStartDelayMsByPath, + this.config.sourceAudioTrackSettings, ), "audio processing", "audio", @@ -1805,6 +1808,7 @@ export class ModernVideoExporter { this.config.audioRegions, this.config.sourceAudioFallbackPaths, this.config.sourceAudioFallbackStartDelayMsByPath, + this.config.sourceAudioTrackSettings, ), description, "audio", diff --git a/src/lib/exporter/sourceTrackRoutingPolicy.test.ts b/src/lib/exporter/sourceTrackRoutingPolicy.test.ts new file mode 100644 index 00000000..ca16cd9b --- /dev/null +++ b/src/lib/exporter/sourceTrackRoutingPolicy.test.ts @@ -0,0 +1,41 @@ +import { describe, expect, it } from "vitest"; +import { resolveSourceTrackRoutingPolicy } from "./sourceTrackRoutingPolicy"; + +describe("resolveSourceTrackRoutingPolicy", () => { + it("prioritizes system+mic sidecars and mutes embedded preview", () => { + const policy = resolveSourceTrackRoutingPolicy("/tmp/recording.mp4", [ + "/tmp/recording.mp4", + "/tmp/recording.system.wav", + "/tmp/recording.mic.wav", + "/tmp/recording.mixed.wav", + ]); + + expect(policy.playbackPaths).toEqual([ + "/tmp/recording.system.wav", + "/tmp/recording.mic.wav", + ]); + expect(policy.muteEmbeddedPreview).toBe(true); + expect(policy.includeEmbeddedInExport).toBe(false); + }); + + it("falls back to mixed when dedicated tracks are absent", () => { + const policy = resolveSourceTrackRoutingPolicy("/tmp/recording.mp4", [ + "/tmp/recording.mixed.wav", + ]); + + expect(policy.playbackPaths).toEqual(["/tmp/recording.mixed.wav"]); + expect(policy.muteEmbeddedPreview).toBe(false); + expect(policy.includeEmbeddedInExport).toBe(false); + }); + + it("keeps embedded audio when only mic sidecar is present", () => { + const policy = resolveSourceTrackRoutingPolicy("/tmp/recording.mp4", [ + "/tmp/recording.mp4", + "/tmp/recording.mic.wav", + ]); + + expect(policy.playbackPaths).toEqual(["/tmp/recording.mic.wav"]); + expect(policy.muteEmbeddedPreview).toBe(true); + expect(policy.includeEmbeddedInExport).toBe(true); + }); +}); diff --git a/src/lib/exporter/sourceTrackRoutingPolicy.ts b/src/lib/exporter/sourceTrackRoutingPolicy.ts new file mode 100644 index 00000000..67601362 --- /dev/null +++ b/src/lib/exporter/sourceTrackRoutingPolicy.ts @@ -0,0 +1,50 @@ +import { resolveSourceAudioFallbackPaths } from "./sourceAudioFallback"; + +export type SourceTrackId = "mic" | "system" | "mixed"; + +export function getSourceTrackIdFromPath(audioPath: string): SourceTrackId { + const normalized = audioPath.toLowerCase(); + if (normalized.includes(".mic.")) return "mic"; + if (normalized.includes(".system.")) return "system"; + return "mixed"; +} + +export interface SourceTrackRoutingPolicy { + hasEmbeddedSourceAudio: boolean; + pathsByTrack: Partial>; + playbackPaths: string[]; + muteEmbeddedPreview: boolean; + includeEmbeddedInExport: boolean; +} + +export function resolveSourceTrackRoutingPolicy( + videoResource: string | null | undefined, + sourceAudioFallbackPaths: string[] | null | undefined, +): SourceTrackRoutingPolicy { + const { hasEmbeddedSourceAudio, externalAudioPaths } = resolveSourceAudioFallbackPaths( + videoResource, + sourceAudioFallbackPaths, + ); + + const pathsByTrack: Partial> = {}; + for (const path of externalAudioPaths) { + const trackId = getSourceTrackIdFromPath(path); + if (!pathsByTrack[trackId]) { + pathsByTrack[trackId] = path; + } + } + + const hasDedicatedTracks = Boolean(pathsByTrack.system || pathsByTrack.mic); + const playbackPaths: string[] = []; + if (pathsByTrack.system) playbackPaths.push(pathsByTrack.system); + if (pathsByTrack.mic) playbackPaths.push(pathsByTrack.mic); + if (!hasDedicatedTracks && pathsByTrack.mixed) playbackPaths.push(pathsByTrack.mixed); + + return { + hasEmbeddedSourceAudio, + pathsByTrack, + playbackPaths, + muteEmbeddedPreview: hasDedicatedTracks, + includeEmbeddedInExport: !pathsByTrack.system && !pathsByTrack.mixed, + }; +} diff --git a/src/lib/exporter/videoExporter.ts b/src/lib/exporter/videoExporter.ts index cf8f6337..55b1b184 100644 --- a/src/lib/exporter/videoExporter.ts +++ b/src/lib/exporter/videoExporter.ts @@ -8,6 +8,7 @@ import type { CursorTelemetryPoint, Padding, SpeedRegion, + SourceAudioTrackSettings, TrimRegion, WebcamOverlaySettings, ZoomMotionBlurTuning, @@ -92,6 +93,7 @@ interface VideoExporterConfig extends ExportConfig { audioRegions?: AudioRegion[]; sourceAudioFallbackPaths?: string[]; sourceAudioFallbackStartDelayMsByPath?: Record; + sourceAudioTrackSettings?: SourceAudioTrackSettings; previewWidth?: number; previewHeight?: number; onProgress?: (progress: ExportProgress) => void; @@ -398,6 +400,7 @@ export class VideoExporter { this.config.audioRegions, this.config.sourceAudioFallbackPaths, this.config.sourceAudioFallbackStartDelayMsByPath, + this.config.sourceAudioTrackSettings, ), "audio processing", "audio", @@ -847,6 +850,7 @@ export class VideoExporter { this.config.audioRegions, this.config.sourceAudioFallbackPaths, this.config.sourceAudioFallbackStartDelayMsByPath, + this.config.sourceAudioTrackSettings, ), "native edited audio rendering", "audio",