From 3211e515fe4b5b5ffccd6a51cb129a5e338e8800 Mon Sep 17 00:00:00 2001 From: Alan Trebugeais Date: Sat, 9 May 2026 20:18:22 +0200 Subject: [PATCH] fix: videoexports with audio (system/mic/mixed/user)... everything should work --- src/components/video-editor/SettingsPanel.tsx | 19 +++-- src/components/video-editor/VideoEditor.tsx | 1 + src/components/video-editor/timeline/Item.tsx | 9 +++ .../components/viewport/TimelineCanvas.tsx | 1 + .../timeline/core/timelineTypes.ts | 1 + .../timeline/model/timelineModel.ts | 1 + src/lib/exporter/audioEncoder.ts | 77 +++++++++++++++++-- src/lib/exporter/modernVideoExporter.ts | 6 +- src/lib/exporter/videoExporter.ts | 7 +- 9 files changed, 105 insertions(+), 17 deletions(-) diff --git a/src/components/video-editor/SettingsPanel.tsx b/src/components/video-editor/SettingsPanel.tsx index f4a8fb22..7b26b4d9 100644 --- a/src/components/video-editor/SettingsPanel.tsx +++ b/src/components/video-editor/SettingsPanel.tsx @@ -3148,14 +3148,19 @@ export function SettingsPanel({ {tSettings("audio.title", "Audio")}
- - {selectedClipMuted - ? tSettings("clip.unmuteAudio", "Unmute Audio") - : tSettings("clip.muteAudio", "Mute Audio")} - +
+ + {tSettings("clip.mute", "Mute")} + +

+ {selectedClipMuted + ? tSettings("clip.mutedState", "Audio is muted") + : tSettings("clip.unmutedState", "Audio is playing")} +

+
onClipMutedChange?.(!v)} + checked={selectedClipMuted ?? false} + onCheckedChange={(v) => onClipMutedChange?.(v)} className="data-[state=checked]:bg-[#06b6d4] scale-75" />
diff --git a/src/components/video-editor/VideoEditor.tsx b/src/components/video-editor/VideoEditor.tsx index 08988c6d..f9c75bbe 100644 --- a/src/components/video-editor/VideoEditor.tsx +++ b/src/components/video-editor/VideoEditor.tsx @@ -4417,6 +4417,7 @@ export default function VideoEditor() { cursorSway, frame, audioRegions, + clipRegions, sourceAudioFallbackPaths: audio.sourceAudioFallbackPaths, sourceAudioFallbackStartDelayMsByPath: audio.sourceAudioFallbackStartDelayMsByPath, diff --git a/src/components/video-editor/timeline/Item.tsx b/src/components/video-editor/timeline/Item.tsx index 3af2ad4d..44405905 100644 --- a/src/components/video-editor/timeline/Item.tsx +++ b/src/components/video-editor/timeline/Item.tsx @@ -5,6 +5,7 @@ import { MusicNotes as Music, MouseLeftClickIcon as PhMouseLeftClick, Scissors, + SpeakerX, MagnifyingGlassPlus as ZoomIn, } from "@phosphor-icons/react"; import type { Span } from "dnd-timeline"; @@ -31,6 +32,7 @@ interface ItemProps { waveformSegmentSpan?: Span; waveformGain?: number; waveformNormalize?: boolean; + muted?: boolean; variant?: "zoom" | "trim" | "clip" | "annotation" | "speed" | "audio"; } @@ -69,6 +71,7 @@ export default function Item({ waveformSegmentSpan, waveformGain = 1, waveformNormalize = false, + muted = false, variant = "zoom", children, }: ItemProps) { @@ -170,6 +173,12 @@ export default function Item({ className="absolute inset-0 w-full h-full pointer-events-none opacity-45" /> )} + {/* Muted overlay for source audio track items */} + {isAudio && muted && ( +
+ +
+ )} {/* Content */}
diff --git a/src/components/video-editor/timeline/components/viewport/TimelineCanvas.tsx b/src/components/video-editor/timeline/components/viewport/TimelineCanvas.tsx index 4d81b323..0c742069 100644 --- a/src/components/video-editor/timeline/components/viewport/TimelineCanvas.tsx +++ b/src/components/video-editor/timeline/components/viewport/TimelineCanvas.tsx @@ -402,6 +402,7 @@ const TimelineCanvasRows = memo(function TimelineCanvasRows({ waveformSegmentSpan={liveSpanPreviewById?.[item.id] ?? item.span} waveformGain={Math.max(0, Math.min(2, settings.volume))} waveformNormalize={Boolean(settings.normalize)} + muted={item.muted} > {track.label} diff --git a/src/components/video-editor/timeline/core/timelineTypes.ts b/src/components/video-editor/timeline/core/timelineTypes.ts index 3464b8ad..573daff4 100644 --- a/src/components/video-editor/timeline/core/timelineTypes.ts +++ b/src/components/video-editor/timeline/core/timelineTypes.ts @@ -39,6 +39,7 @@ export interface TimelineRenderItem { zoomMode?: ZoomMode; speedValue?: number; showSourceAudio?: boolean; + muted?: boolean; variant: "zoom" | "trim" | "clip" | "annotation" | "speed" | "audio"; } diff --git a/src/components/video-editor/timeline/model/timelineModel.ts b/src/components/video-editor/timeline/model/timelineModel.ts index 05073c97..70f63c4c 100644 --- a/src/components/video-editor/timeline/model/timelineModel.ts +++ b/src/components/video-editor/timeline/model/timelineModel.ts @@ -53,6 +53,7 @@ export function buildTimelineItems(params: { span: { start: region.startMs, end: region.endMs }, label: `Clip ${index + 1}`, showSourceAudio: region.showSourceAudio, + muted: Boolean(region.muted), variant: "clip", })); diff --git a/src/lib/exporter/audioEncoder.ts b/src/lib/exporter/audioEncoder.ts index 0e342c76..ec42cf83 100644 --- a/src/lib/exporter/audioEncoder.ts +++ b/src/lib/exporter/audioEncoder.ts @@ -47,6 +47,7 @@ interface PreparedOfflineRender { mainBufferEntry: { buffer: AudioBuffer; gain: number } | null; companionEntries: Array<{ buffer: AudioBuffer; startDelaySec: number; gain: number }>; regionEntries: Array<{ buffer: AudioBuffer; region: AudioRegion }>; + mutedSourceOutputRangesSec: Array<{ startSec: number; endSec: number }>; slices: TimelineSlice[]; outputDurationMs: number; numChannels: number; @@ -163,6 +164,7 @@ export class AudioProcessor { sourceAudioFallbackPaths?: string[], sourceAudioFallbackStartDelayMsByPath?: Record, sourceAudioTrackSettings?: SourceAudioTrackSettings, + clipRegions?: ClipRegion[], ): Promise { const sortedTrims = trimRegions ? [...trimRegions].sort((a, b) => a.startMs - b.startMs) @@ -206,6 +208,7 @@ export class AudioProcessor { sortedSourceAudioFallbackPaths, sourceAudioFallbackStartDelayMsByPath, sourceAudioTrackSettings, + clipRegions, muxer, ); return; @@ -238,6 +241,7 @@ export class AudioProcessor { routingPolicy.playbackPaths, sourceAudioFallbackStartDelayMsByPath, sourceAudioTrackSettings, + clipRegions, muxer, ); return; @@ -285,6 +289,7 @@ export class AudioProcessor { sourceAudioFallbackPaths?: string[], sourceAudioFallbackStartDelayMsByPath?: Record, sourceAudioTrackSettings?: SourceAudioTrackSettings, + clipRegions?: ClipRegion[], ): Promise { const sortedTrims = trimRegions ? [...trimRegions].sort((a, b) => a.startMs - b.startMs) @@ -311,6 +316,7 @@ export class AudioProcessor { sortedSourceAudioFallbackPaths, sourceAudioFallbackStartDelayMsByPath, sourceAudioTrackSettings, + clipRegions, ); return this.renderToWavBlobChunked(prepared); } @@ -587,6 +593,7 @@ export class AudioProcessor { sourceAudioFallbackPaths: string[], sourceAudioFallbackStartDelayMsByPath: Record | undefined, sourceAudioTrackSettings: SourceAudioTrackSettings | undefined, + clipRegions: ClipRegion[] | undefined, muxer: VideoMuxer, ): Promise { const prepared = await this.prepareOfflineRender( @@ -597,6 +604,7 @@ export class AudioProcessor { sourceAudioFallbackPaths, sourceAudioFallbackStartDelayMsByPath, sourceAudioTrackSettings, + clipRegions, ); if (this.cancelled) return; await this.renderAndEncodeChunked(prepared, muxer); @@ -610,6 +618,7 @@ export class AudioProcessor { sourceAudioFallbackPaths: string[], sourceAudioFallbackStartDelayMsByPath?: Record, sourceAudioTrackSettings?: SourceAudioTrackSettings, + clipRegions?: ClipRegion[], ): Promise { if (this.cancelled) throw new Error("Export cancelled"); this.onProgress?.(0); @@ -711,11 +720,24 @@ export class AudioProcessor { } const numChannels = Math.min(primaryBuffer?.numberOfChannels ?? 2, 2); + const mutedSourceOutputRangesSec = (clipRegions ?? []) + .filter( + (clip) => + Boolean(clip.muted) && + Number.isFinite(clip.startMs) && + Number.isFinite(clip.endMs) && + clip.endMs > clip.startMs, + ) + .map((clip) => ({ + startSec: Math.max(0, clip.startMs / 1000), + endSec: Math.max(0, clip.endMs / 1000), + })); return { mainBufferEntry, companionEntries, regionEntries, + mutedSourceOutputRangesSec, slices, outputDurationMs, numChannels, @@ -849,6 +871,7 @@ export class AudioProcessor { prepared.mainBufferEntry.gain, outputOffsetSec, chunkSec, + prepared.mutedSourceOutputRangesSec, ); } @@ -862,6 +885,7 @@ export class AudioProcessor { entry.gain, outputOffsetSec, chunkSec, + prepared.mutedSourceOutputRangesSec, ); } @@ -1323,6 +1347,7 @@ export class AudioProcessor { gain = 1, chunkOutputStartSec = 0, chunkDurationSec = Number.POSITIVE_INFINITY, + mutedOutputRangesSec: Array<{ startSec: number; endSec: number }> = [], ): void { let outputOffsetSec = 0; @@ -1386,15 +1411,51 @@ export class AudioProcessor { continue; } - const source = ctx.createBufferSource(); - const gainNode = ctx.createGain(); - gainNode.gain.value = Math.max(0, Math.min(2, gain)); - source.buffer = buffer; - source.playbackRate.value = slice.speed; - source.connect(gainNode); - gainNode.connect(ctx.destination); + const audibleRanges: Array<{ startSec: number; endSec: number }> = [ + { + startSec: localOutputStartSec + chunkOutputStartSec, + endSec: + localOutputStartSec + chunkOutputStartSec + effectiveSourceDurationSec / slice.speed, + }, + ]; + for (const mutedRange of mutedOutputRangesSec) { + for (let rangeIndex = audibleRanges.length - 1; rangeIndex >= 0; rangeIndex -= 1) { + const current = audibleRanges[rangeIndex]; + const overlapStart = Math.max(current.startSec, mutedRange.startSec); + const overlapEnd = Math.min(current.endSec, mutedRange.endSec); + if (overlapEnd <= overlapStart) { + continue; + } + audibleRanges.splice(rangeIndex, 1); + if (current.startSec < overlapStart) { + audibleRanges.push({ startSec: current.startSec, endSec: overlapStart }); + } + if (overlapEnd < current.endSec) { + audibleRanges.push({ startSec: overlapEnd, endSec: current.endSec }); + } + } + } - source.start(localOutputStartSec, effectiveBufferStartSec, effectiveSourceDurationSec); + for (const audibleRange of audibleRanges) { + const audibleDurationSec = audibleRange.endSec - audibleRange.startSec; + if (audibleDurationSec <= 0.001) { + continue; + } + const source = ctx.createBufferSource(); + const gainNode = ctx.createGain(); + gainNode.gain.value = Math.max(0, Math.min(2, gain)); + source.buffer = buffer; + source.playbackRate.value = slice.speed; + source.connect(gainNode); + gainNode.connect(ctx.destination); + + const sourceOffsetSec = + effectiveBufferStartSec + + (audibleRange.startSec - (localOutputStartSec + chunkOutputStartSec)) * slice.speed; + const localStartSec = audibleRange.startSec - chunkOutputStartSec; + const sourceDurationSec = audibleDurationSec * slice.speed; + source.start(localStartSec, sourceOffsetSec, sourceDurationSec); + } outputOffsetSec += sliceOutputDurationSec; } diff --git a/src/lib/exporter/modernVideoExporter.ts b/src/lib/exporter/modernVideoExporter.ts index 3967d35a..bca2c4dd 100644 --- a/src/lib/exporter/modernVideoExporter.ts +++ b/src/lib/exporter/modernVideoExporter.ts @@ -3,6 +3,7 @@ import type { AudioRegion, AutoCaptionSettings, CaptionCue, + ClipRegion, CropRegion, CursorStyle, CursorTelemetryPoint, @@ -136,6 +137,7 @@ interface VideoExporterConfig extends ExportConfig { zoomClassicMode?: boolean; frame?: string | null; audioRegions?: AudioRegion[]; + clipRegions?: ClipRegion[]; sourceAudioFallbackPaths?: string[]; sourceAudioFallbackStartDelayMsByPath?: Record; sourceAudioTrackSettings?: SourceAudioTrackSettings; @@ -1195,7 +1197,8 @@ export class ModernVideoExporter { audioRegions.length > 0 || sourceAudioFallbackPaths.length > 1 || hasTimedSourceAudioFallback || - hasNonDefaultSourceTrackSettings(this.config.sourceAudioTrackSettings) + hasNonDefaultSourceTrackSettings(this.config.sourceAudioTrackSettings) || + (this.config.clipRegions ?? []).some((clip) => Boolean(clip.muted)) ) { const sourceDurationMs = Math.max( 0, @@ -1820,6 +1823,7 @@ export class ModernVideoExporter { this.config.sourceAudioFallbackPaths, this.config.sourceAudioFallbackStartDelayMsByPath, this.config.sourceAudioTrackSettings, + this.config.clipRegions, ), description, "audio", diff --git a/src/lib/exporter/videoExporter.ts b/src/lib/exporter/videoExporter.ts index 09dea96f..b99f5f96 100644 --- a/src/lib/exporter/videoExporter.ts +++ b/src/lib/exporter/videoExporter.ts @@ -3,6 +3,7 @@ import type { AudioRegion, AutoCaptionSettings, CaptionCue, + ClipRegion, CropRegion, CursorStyle, CursorTelemetryPoint, @@ -91,6 +92,7 @@ interface VideoExporterConfig extends ExportConfig { zoomSmoothness?: number; frame?: string | null; audioRegions?: AudioRegion[]; + clipRegions?: ClipRegion[]; sourceAudioFallbackPaths?: string[]; sourceAudioFallbackStartDelayMsByPath?: Record; sourceAudioTrackSettings?: SourceAudioTrackSettings; @@ -574,7 +576,8 @@ export class VideoExporter { audioRegions.length > 0 || sourceAudioFallbackPaths.length > 1 || hasTimedSourceAudioFallback || - hasNonDefaultSourceTrackSettings(this.config.sourceAudioTrackSettings) + hasNonDefaultSourceTrackSettings(this.config.sourceAudioTrackSettings) || + (this.config.clipRegions ?? []).some((clip) => Boolean(clip.muted)) ) { const sourceDurationMs = Math.max( 0, @@ -862,6 +865,7 @@ export class VideoExporter { this.config.sourceAudioFallbackPaths, this.config.sourceAudioFallbackStartDelayMsByPath, this.config.sourceAudioTrackSettings, + this.config.clipRegions, ), "native edited audio rendering", "audio", @@ -959,6 +963,7 @@ export class VideoExporter { this.config.sourceAudioFallbackPaths, this.config.sourceAudioFallbackStartDelayMsByPath, this.config.sourceAudioTrackSettings, + this.config.clipRegions, ), "ffmpeg edited audio rendering", "audio",