From eff3bbf2f77455c9eb365f0771aec227fcdd70f5 Mon Sep 17 00:00:00 2001 From: Alan Trebugeais Date: Sat, 9 May 2026 13:29:06 +0200 Subject: [PATCH 01/25] fix: audio waveform on the wrong layer and not affected by clip size/region --- src/components/video-editor/timeline/Item.tsx | 15 ++ .../video-editor/timeline/TimelineEditor.tsx | 71 ++++++ .../components/viewport/TimelineCanvas.tsx | 42 +++- .../components/waveform/AudioWaveform.tsx | 18 +- .../components/wrapper/TimelineWrapper.tsx | 17 +- .../timeline/hooks/useTimelineAudioPeaks.ts | 210 +++++++++++++++--- 6 files changed, 323 insertions(+), 50 deletions(-) diff --git a/src/components/video-editor/timeline/Item.tsx b/src/components/video-editor/timeline/Item.tsx index 333b8855..33703a82 100644 --- a/src/components/video-editor/timeline/Item.tsx +++ b/src/components/video-editor/timeline/Item.tsx @@ -11,6 +11,8 @@ import type { Span } from "dnd-timeline"; import { useItem } from "dnd-timeline"; import { useMemo } from "react"; import { cn } from "@/lib/utils"; +import AudioWaveform from "./components/waveform/AudioWaveform"; +import type { AudioPeaksData } from "./core/timelineTypes"; import glassStyles from "./ItemGlass.module.css"; interface ItemProps { @@ -24,6 +26,8 @@ interface ItemProps { zoomDepth?: number; zoomMode?: "auto" | "manual"; speedValue?: number; + waveformPeaks?: AudioPeaksData | null; + waveformSegmentSpan?: Span; variant?: "zoom" | "trim" | "clip" | "annotation" | "speed" | "audio"; } @@ -57,6 +61,8 @@ export default function Item({ zoomDepth = 1, zoomMode = "auto", speedValue, + waveformPeaks = null, + waveformSegmentSpan, variant = "zoom", children, }: ItemProps) { @@ -71,6 +77,7 @@ export default function Item({ const isClip = variant === "clip"; const isSpeed = variant === "speed"; const isAudio = variant === "audio"; + const showAudioWaveform = isAudio && Boolean(waveformPeaks); const glassClass = isZoom ? glassStyles.glassPurple @@ -146,6 +153,14 @@ export default function Item({ style={{ cursor: "col-resize", pointerEvents: "auto" }} title="Resize right" /> + {showAudioWaveform && waveformPeaks && ( + + )} {/* Content */}
diff --git a/src/components/video-editor/timeline/TimelineEditor.tsx b/src/components/video-editor/timeline/TimelineEditor.tsx index 818caa42..29b0f25c 100644 --- a/src/components/video-editor/timeline/TimelineEditor.tsx +++ b/src/components/video-editor/timeline/TimelineEditor.tsx @@ -178,6 +178,56 @@ const TimelineEditor = forwardRef( pan: "Shift + Ctrl + Scroll", zoom: "Ctrl + Scroll", }); + const [liveSpanPreviewById, setLiveSpanPreviewById] = useState>({}); + const liveZoomPreview = useMemo(() => { + const previewSpans: Record = { ...liveSpanPreviewById }; + const hiddenZoomIds = new Set(); + + for (const [previewId, previewSpan] of Object.entries(liveSpanPreviewById)) { + const oldClip = clipRegions.find((clip) => clip.id === previewId); + if (!oldClip) continue; + + const newStart = Math.round(previewSpan.start); + const newEnd = Math.round(previewSpan.end); + const removedSegments = [ + ...(newStart > oldClip.startMs + ? [{ startMs: oldClip.startMs, endMs: newStart }] + : []), + ...(newEnd < oldClip.endMs + ? [{ startMs: newEnd, endMs: oldClip.endMs }] + : []), + ]; + + const startDelta = newStart - oldClip.startMs; + const endDelta = newEnd - oldClip.endMs; + const isMove = Math.abs(startDelta - endDelta) < 1 && Math.abs(startDelta) > 0; + + if (isMove) { + const delta = startDelta; + for (const zoom of zoomRegions) { + const overlaps = + zoom.startMs < oldClip.endMs && zoom.endMs > oldClip.startMs; + if (!overlaps) continue; + previewSpans[zoom.id] = { + start: zoom.startMs + delta, + end: zoom.endMs + delta, + }; + } + } + + if (removedSegments.length > 0) { + for (const zoom of zoomRegions) { + const removed = removedSegments.some( + (segment) => + zoom.startMs < segment.endMs && zoom.endMs > segment.startMs, + ); + if (removed) hiddenZoomIds.add(zoom.id); + } + } + } + + return { previewSpans, hiddenZoomIds }; + }, [clipRegions, liveSpanPreviewById, zoomRegions]); const { shortcuts: keyShortcuts, isMac } = useShortcuts(); const audioPeaks = useTimelineAudioPeaks(videoPath); @@ -376,6 +426,25 @@ const TimelineEditor = forwardRef( onItemSpanChange={handleItemSpanChange} resolveTargetRowId={getResolvedDropRowId} allRegionSpans={allRegionSpans} + onLiveSpanPreviewChange={(id, span) => { + setLiveSpanPreviewById((prev) => { + if (!span) { + if (!(id in prev)) return prev; + const next = { ...prev }; + delete next[id]; + return next; + } + const current = prev[id]; + if ( + current && + current.start === span.start && + current.end === span.end + ) { + return prev; + } + return { ...prev, [id]: span }; + }); + }} > ( onClearBlockSelection={clearSelectedBlocks} keyframes={keyframes} audioPeaks={audioPeaks} + liveSpanPreviewById={liveZoomPreview.previewSpans} + liveHiddenItemIds={Array.from(liveZoomPreview.hiddenZoomIds)} />
diff --git a/src/components/video-editor/timeline/components/viewport/TimelineCanvas.tsx b/src/components/video-editor/timeline/components/viewport/TimelineCanvas.tsx index 48e29b6f..d6d13ed1 100644 --- a/src/components/video-editor/timeline/components/viewport/TimelineCanvas.tsx +++ b/src/components/video-editor/timeline/components/viewport/TimelineCanvas.tsx @@ -17,7 +17,6 @@ import { getTimelineViewportStretchFactor, TIMELINE_AXIS_HEIGHT_PX, } from "../../timelineLayout"; -import AudioWaveform from "../waveform/AudioWaveform"; import glassStyles from "../../ItemGlass.module.css"; import Item from "../../Item"; import Row from "../../Row"; @@ -38,6 +37,7 @@ import PlaybackCursor from "../playhead/PlaybackCursor"; const HINT_CLIP = "Press C to split clip"; const HINT_ANNOTATION = "Press A to add annotation"; const HINT_AUDIO = "Click music icon to add audio"; +const SOURCE_AUDIO_ROW_ID = "row-source-audio"; interface TimelineCanvasProps { items: TimelineRenderItem[]; @@ -58,6 +58,8 @@ interface TimelineCanvasProps { onClearBlockSelection?: () => void; keyframes?: { id: string; time: number }[]; audioPeaks?: AudioPeaksData | null; + liveSpanPreviewById?: Record; + liveHiddenItemIds?: string[]; } interface TimelineHoverParams { @@ -219,6 +221,8 @@ interface TimelineCanvasRowsProps { onSelectAnnotation?: (id: string | null) => void; onSelectAudio?: (id: string | null) => void; audioPeaks?: AudioPeaksData | null; + liveSpanPreviewById?: Record; + liveHiddenItemIds?: string[]; direction: string; canShowGhostZoom: boolean; ghostStartMs: number | null; @@ -243,6 +247,8 @@ const TimelineCanvasRows = memo(function TimelineCanvasRows({ onSelectAnnotation, onSelectAudio, audioPeaks, + liveSpanPreviewById, + liveHiddenItemIds, direction, canShowGhostZoom, ghostStartMs, @@ -253,6 +259,7 @@ const TimelineCanvasRows = memo(function TimelineCanvasRows({ onZoomRowMouseLeave, onZoomRowClick, }: TimelineCanvasRowsProps) { + const hiddenIds = useMemo(() => new Set(liveHiddenItemIds ?? []), [liveHiddenItemIds]); const { clipItems, zoomItems, annotationRows, audioRows } = useMemo(() => { const nextClipItems: TimelineRenderItem[] = []; const nextZoomItems: TimelineRenderItem[] = []; @@ -307,7 +314,6 @@ const TimelineCanvasRows = memo(function TimelineCanvasRows({ return ( <> - {audioPeaks && } {clipItems.map((item) => ( ))} + {audioPeaks && ( + + {clipItems.map((item) => ( + onSelectClip?.(item.id)} + variant="audio" + waveformPeaks={audioPeaks} + waveformSegmentSpan={liveSpanPreviewById?.[item.id] ?? item.span} + > + Source + + ))} + + )}
)} - {zoomItems.map((item) => ( + {zoomItems + .filter((item) => !hiddenIds.has(item.id)) + .map((item) => ( (null); const { range } = useTimelineContext(); const [resizeKey, setResizeKey] = useState(0); @@ -53,8 +61,8 @@ function AudioWaveformComponent({ peaks }: AudioWaveformProps) { const { peaks: peakData, durationMs } = peaks; if (durationMs <= 0 || peakData.length === 0) return; - const visibleStartMs = range.start; - const visibleEndMs = range.end; + const visibleStartMs = segmentStartMs ?? range.start; + const visibleEndMs = segmentEndMs ?? range.end; const visibleDurationMs = visibleEndMs - visibleStartMs; if (visibleDurationMs <= 0) return; @@ -77,12 +85,12 @@ function AudioWaveformComponent({ peaks }: AudioWaveformProps) { ctx.strokeStyle = "rgba(255, 255, 255, 0.55)"; ctx.lineWidth = dpr; ctx.stroke(); - }, [peaks, range.start, range.end, resizeKey]); + }, [peaks, range.start, range.end, resizeKey, segmentStartMs, segmentEndMs]); return ( ); diff --git a/src/components/video-editor/timeline/components/wrapper/TimelineWrapper.tsx b/src/components/video-editor/timeline/components/wrapper/TimelineWrapper.tsx index 5e777e70..917b171a 100644 --- a/src/components/video-editor/timeline/components/wrapper/TimelineWrapper.tsx +++ b/src/components/video-editor/timeline/components/wrapper/TimelineWrapper.tsx @@ -29,6 +29,7 @@ interface TimelineWrapperProps { onItemSpanChange: (id: string, span: Span, rowId?: string) => void; resolveTargetRowId?: (id: string, proposedRowId: string) => string; allRegionSpans?: TimelineRegionSpan[]; + onLiveSpanPreviewChange?: (id: string, span: Span | null) => void; } export default function TimelineWrapper({ @@ -43,6 +44,7 @@ export default function TimelineWrapper({ onItemSpanChange, resolveTargetRowId, allRegionSpans = [], + onLiveSpanPreviewChange, }: TimelineWrapperProps) { const totalMs = Math.max(0, Math.round(videoDuration * 1000)); @@ -132,8 +134,9 @@ export default function TimelineWrapper({ (event: DragStartEvent) => { const span = event.active.data.current.getSpanFromDragEvent?.(event); if (span) showTooltip(span); + onLiveSpanPreviewChange?.(event.active.id as string, span ?? null); }, - [showTooltip], + [onLiveSpanPreviewChange, showTooltip], ); const onDragMove = useCallback( @@ -144,8 +147,9 @@ export default function TimelineWrapper({ ? (event.activatorEvent as PointerEvent).clientX + (event.delta?.x ?? 0) : undefined; if (span) showTooltip(span, screenX); + onLiveSpanPreviewChange?.(event.active.id as string, span ?? null); }, - [showTooltip], + [onLiveSpanPreviewChange, showTooltip], ); const onResizeMove = useCallback( @@ -156,8 +160,9 @@ export default function TimelineWrapper({ ? (event.activatorEvent as PointerEvent).clientX + (event.delta?.x ?? 0) : undefined; if (span) showTooltip(span, screenX); + onLiveSpanPreviewChange?.(event.active.id as string, span ?? null); }, - [showTooltip], + [onLiveSpanPreviewChange, showTooltip], ); const hideTooltip = useCallback(() => showTooltip(null), [showTooltip]); @@ -166,16 +171,18 @@ export default function TimelineWrapper({ (event: ResizeEndEvent) => { hideTooltip(); onResizeEnd(event); + onLiveSpanPreviewChange?.(event.active.id as string, null); }, - [hideTooltip, onResizeEnd], + [hideTooltip, onLiveSpanPreviewChange, onResizeEnd], ); const onDragEndWithTooltip = useCallback( (event: DragEndEvent) => { hideTooltip(); onDragEnd(event); + onLiveSpanPreviewChange?.(event.active.id as string, null); }, - [hideTooltip, onDragEnd], + [hideTooltip, onDragEnd, onLiveSpanPreviewChange], ); const handleRangeChange = useCallback( diff --git a/src/components/video-editor/timeline/hooks/useTimelineAudioPeaks.ts b/src/components/video-editor/timeline/hooks/useTimelineAudioPeaks.ts index ed377663..9af79a80 100644 --- a/src/components/video-editor/timeline/hooks/useTimelineAudioPeaks.ts +++ b/src/components/video-editor/timeline/hooks/useTimelineAudioPeaks.ts @@ -1,9 +1,104 @@ import { useEffect, useRef, useState } from "react"; +import { fromFileUrl, resolveVideoUrl } from "../../projectPersistence"; import type { AudioPeaksData } from "../core/timelineTypes"; /** Number of peak bins to produce — enough for smooth display at any zoom. */ const TARGET_PEAK_COUNT = 2048; +function buildSidecarAudioCandidates(sourcePath: string): string[] { + const normalized = sourcePath.replace(/\\/g, "/"); + const lastSlash = normalized.lastIndexOf("/"); + const dir = lastSlash >= 0 ? normalized.slice(0, lastSlash + 1) : ""; + const fileName = lastSlash >= 0 ? normalized.slice(lastSlash + 1) : normalized; + const dotIndex = fileName.lastIndexOf("."); + const baseName = dotIndex > 0 ? fileName.slice(0, dotIndex) : fileName; + + return [ + `${dir}${baseName}.system.wav`, + `${dir}${baseName}.mic.wav`, + `${dir}${baseName}.system.m4a`, + `${dir}${baseName}.mic.m4a`, + ]; +} + +function extractLocalPathFromMediaServerUrl(input: string): string | null { + try { + const url = new URL(input); + const isLocalMediaServer = + (url.protocol === "http:" || url.protocol === "https:") && + (url.hostname === "127.0.0.1" || url.hostname === "localhost") && + url.pathname === "/video"; + if (!isLocalMediaServer) return null; + const rawPath = url.searchParams.get("path"); + if (!rawPath) return null; + return rawPath; + } catch { + return null; + } +} + +async function decodePeaksFromMediaUrl( + mediaUrl: string, + cancelled: () => boolean, +): Promise { + const response = await fetch(mediaUrl); + if (!response.ok) { + throw new Error(`Failed to load media for waveform: ${response.status}`); + } + if (cancelled()) throw new Error("cancelled"); + console.debug("[timeline-audio-peaks] fetch ok", { + url: mediaUrl, + status: response.status, + }); + + const arrayBuffer = await response.arrayBuffer(); + if (cancelled()) throw new Error("cancelled"); + console.debug("[timeline-audio-peaks] bytes loaded", { + url: mediaUrl, + byteLength: arrayBuffer.byteLength, + }); + + const audioCtx = new OfflineAudioContext(1, 1, 44100); + const decoded = await audioCtx.decodeAudioData(arrayBuffer); + if (cancelled()) throw new Error("cancelled"); + console.debug("[timeline-audio-peaks] decode ok", { + url: mediaUrl, + durationSec: decoded.duration, + channelCount: decoded.numberOfChannels, + sampleRate: decoded.sampleRate, + sampleFrames: decoded.length, + }); + + const channelData = decoded.getChannelData(0); + const durationMs = decoded.duration * 1000; + const binSize = Math.max(1, Math.floor(channelData.length / TARGET_PEAK_COUNT)); + const peakCount = Math.ceil(channelData.length / binSize); + const peaks = new Float32Array(peakCount); + + for (let i = 0; i < peakCount; i++) { + const start = i * binSize; + const end = Math.min(start + binSize, channelData.length); + let max = 0; + for (let j = start; j < end; j++) { + const abs = Math.abs(channelData[j]); + if (abs > max) max = abs; + } + peaks[i] = max; + } + + let globalMax = 0; + for (let i = 0; i < peaks.length; i++) { + if (peaks[i] > globalMax) globalMax = peaks[i]; + } + if (globalMax > 0) { + for (let i = 0; i < peaks.length; i++) { + peaks[i] /= globalMax; + } + } + + return { peaks, durationMs }; +} + /** * Decode audio from a media file URL and produce a fixed-length array of peak * amplitudes suitable for waveform visualisation. @@ -17,8 +112,10 @@ export function useTimelineAudioPeaks(fileUrl: string | null | undefined): Audio useEffect(() => { urlRef.current = fileUrl; setData(null); + console.debug("[timeline-audio-peaks] reset", { input: fileUrl ?? null }); if (!fileUrl) { + console.debug("[timeline-audio-peaks] no input source, skipping"); return; } @@ -26,49 +123,92 @@ export function useTimelineAudioPeaks(fileUrl: string | null | undefined): Audio (async () => { try { - const response = await fetch(fileUrl); + const extractedLocalPath = extractLocalPathFromMediaServerUrl(fileUrl); + const isRemoteLike = /^(https?:|blob:|data:)/i.test(fileUrl) && !extractedLocalPath; + const isFileUrl = /^file:\/\//i.test(fileUrl); + const sourceLocalPath = isRemoteLike + ? null + : extractedLocalPath + ? extractedLocalPath + : isFileUrl + ? fromFileUrl(fileUrl) + : fileUrl; + console.debug("[timeline-audio-peaks] resolving source", { + input: fileUrl, + isRemoteLike, + isFileUrl, + sourceLocalPath, + extractedLocalPath, + }); + const mediaUrl = isRemoteLike + ? fileUrl + : isFileUrl + ? await resolveVideoUrl(fromFileUrl(fileUrl)) + : await resolveVideoUrl(fileUrl); if (cancelled) return; + console.debug("[timeline-audio-peaks] resolved source", { + input: fileUrl, + resolved: mediaUrl, + }); - const arrayBuffer = await response.arrayBuffer(); - if (cancelled) return; - - const audioCtx = new OfflineAudioContext(1, 1, 44100); - const decoded = await audioCtx.decodeAudioData(arrayBuffer); - if (cancelled) return; - - const channelData = decoded.getChannelData(0); - const durationMs = decoded.duration * 1000; - const binSize = Math.max(1, Math.floor(channelData.length / TARGET_PEAK_COUNT)); - const peakCount = Math.ceil(channelData.length / binSize); - const peaks = new Float32Array(peakCount); - - for (let i = 0; i < peakCount; i++) { - const start = i * binSize; - const end = Math.min(start + binSize, channelData.length); - let max = 0; - for (let j = start; j < end; j++) { - const abs = Math.abs(channelData[j]); - if (abs > max) max = abs; + let decodedPeaks: AudioPeaksData | null = null; + try { + decodedPeaks = await decodePeaksFromMediaUrl(mediaUrl, () => cancelled); + } catch (primaryError) { + if (primaryError instanceof Error && primaryError.message === "cancelled") { + return; } - peaks[i] = max; - } - - // Normalise to 0–1 range. - let globalMax = 0; - for (let i = 0; i < peaks.length; i++) { - if (peaks[i] > globalMax) globalMax = peaks[i]; - } - if (globalMax > 0) { - for (let i = 0; i < peaks.length; i++) { - peaks[i] /= globalMax; + console.warn("[timeline-audio-peaks] primary decode failed", { + input: fileUrl, + url: mediaUrl, + error: primaryError, + }); + if (sourceLocalPath) { + const candidates = buildSidecarAudioCandidates(sourceLocalPath); + console.debug("[timeline-audio-peaks] trying sidecar candidates", { + sourceLocalPath, + candidates, + }); + for (const candidatePath of candidates) { + try { + const candidateUrl = await resolveVideoUrl(candidatePath); + console.debug("[timeline-audio-peaks] trying sidecar", { + candidatePath, + candidateUrl, + }); + decodedPeaks = await decodePeaksFromMediaUrl(candidateUrl, () => cancelled); + console.info("[timeline-audio-peaks] sidecar decode succeeded", { + candidatePath, + candidateUrl, + }); + break; + } catch (sidecarError) { + if (sidecarError instanceof Error && sidecarError.message === "cancelled") { + return; + } + console.debug("[timeline-audio-peaks] sidecar decode failed", { + candidatePath, + error: sidecarError, + }); + } + } } } - if (!cancelled && urlRef.current === fileUrl) { - setData({ peaks, durationMs }); + if (!cancelled && urlRef.current === fileUrl && decodedPeaks) { + console.debug("[timeline-audio-peaks] peaks generated", { + input: fileUrl, + durationMs: Math.round(decodedPeaks.durationMs), + peakCount: decodedPeaks.peaks.length, + }); + setData(decodedPeaks); } - } catch { + } catch (error) { // File has no audio or decoding failed — leave as null. + console.warn("[timeline-audio-peaks] failed", { + input: fileUrl, + error, + }); } })(); From 29e10b9ec1a241bdaa5a4c2b042702d9880cf1cf Mon Sep 17 00:00:00 2001 From: Alan Trebugeais Date: Sat, 9 May 2026 13:35:41 +0200 Subject: [PATCH 02/25] add: TimelineEditor waveformgenerator as a worker. to be extra efficient --- .../video-editor/timeline/TimelineEditor.tsx | 4 +- .../components/viewport/TimelineCanvas.tsx | 46 +++- .../components/waveform/AudioWaveform.tsx | 20 +- .../components/waveform/WaveformGenerator.ts | 88 +++++++ .../components/waveform/waveform.worker.ts | 31 +++ .../timeline/core/timelineTypes.ts | 1 + .../timeline/hooks/useTimelineAudioPeaks.ts | 217 ++++-------------- .../timeline/model/timelineModel.ts | 1 + 8 files changed, 222 insertions(+), 186 deletions(-) create mode 100644 src/components/video-editor/timeline/components/waveform/WaveformGenerator.ts create mode 100644 src/components/video-editor/timeline/components/waveform/waveform.worker.ts diff --git a/src/components/video-editor/timeline/TimelineEditor.tsx b/src/components/video-editor/timeline/TimelineEditor.tsx index 29b0f25c..cf7005e4 100644 --- a/src/components/video-editor/timeline/TimelineEditor.tsx +++ b/src/components/video-editor/timeline/TimelineEditor.tsx @@ -229,7 +229,9 @@ const TimelineEditor = forwardRef( return { previewSpans, hiddenZoomIds }; }, [clipRegions, liveSpanPreviewById, zoomRegions]); const { shortcuts: keyShortcuts, isMac } = useShortcuts(); - const audioPeaks = useTimelineAudioPeaks(videoPath); + const audioPeaks = useTimelineAudioPeaks(videoPath, { + enableSourceSidecarFallback: true, + }); useEffect(() => { if (aspectRatio === "native") { diff --git a/src/components/video-editor/timeline/components/viewport/TimelineCanvas.tsx b/src/components/video-editor/timeline/components/viewport/TimelineCanvas.tsx index d6d13ed1..c7d65898 100644 --- a/src/components/video-editor/timeline/components/viewport/TimelineCanvas.tsx +++ b/src/components/video-editor/timeline/components/viewport/TimelineCanvas.tsx @@ -33,6 +33,7 @@ import { import TimelineAxis from "../axis/TimelineAxis"; import ClipMarkerOverlay from "../overlays/ClipMarkerOverlay"; import PlaybackCursor from "../playhead/PlaybackCursor"; +import { useTimelineAudioPeaks } from "../../hooks/useTimelineAudioPeaks"; const HINT_CLIP = "Press C to split clip"; const HINT_ANNOTATION = "Press A to add annotation"; @@ -234,6 +235,38 @@ interface TimelineCanvasRowsProps { onZoomRowClick: MouseEventHandler; } +interface AudioItemWithWaveformProps { + item: TimelineRenderItem; + span: { start: number; end: number }; + waveformSpan: { start: number; end: number }; + isSelected: boolean; + onSelectAudio?: (id: string | null) => void; +} + +function AudioItemWithWaveform({ + item, + span, + waveformSpan, + isSelected, + onSelectAudio, +}: AudioItemWithWaveformProps) { + const peaks = useTimelineAudioPeaks(item.audioPath ?? null); + return ( + + {item.label} + + ); +} + const TimelineCanvasRows = memo(function TimelineCanvasRows({ items, videoDurationMs, @@ -422,17 +455,14 @@ const TimelineCanvasRows = memo(function TimelineCanvasRows({ {audioRows.map(({ rowId, items: rowItems }, index) => ( {rowItems.map((item) => ( - - {item.label} - + onSelectAudio={onSelectAudio} + /> ))} ))} diff --git a/src/components/video-editor/timeline/components/waveform/AudioWaveform.tsx b/src/components/video-editor/timeline/components/waveform/AudioWaveform.tsx index 367ed1ed..eff26d09 100644 --- a/src/components/video-editor/timeline/components/waveform/AudioWaveform.tsx +++ b/src/components/video-editor/timeline/components/waveform/AudioWaveform.tsx @@ -61,8 +61,13 @@ function AudioWaveformComponent({ const { peaks: peakData, durationMs } = peaks; if (durationMs <= 0 || peakData.length === 0) return; - const visibleStartMs = segmentStartMs ?? range.start; - const visibleEndMs = segmentEndMs ?? range.end; + const rawVisibleStartMs = segmentStartMs ?? range.start; + const rawVisibleEndMs = segmentEndMs ?? range.end; + const msPerBin = durationMs / peakData.length; + const visibleStartMs = + msPerBin > 0 ? Math.round(rawVisibleStartMs / msPerBin) * msPerBin : rawVisibleStartMs; + const visibleEndMs = + msPerBin > 0 ? Math.round(rawVisibleEndMs / msPerBin) * msPerBin : rawVisibleEndMs; const visibleDurationMs = visibleEndMs - visibleStartMs; if (visibleDurationMs <= 0) return; @@ -71,11 +76,14 @@ function AudioWaveformComponent({ ctx.beginPath(); for (let px = 0; px < width; px++) { const t = visibleStartMs + (px / width) * visibleDurationMs; - const binIndex = Math.min( - peakData.length - 1, - Math.max(0, Math.floor((t / durationMs) * peakData.length)), + const exactIndex = Math.max( + 0, + Math.min(peakData.length - 1, (t / durationMs) * (peakData.length - 1)), ); - const amplitude = peakData[binIndex]; + const leftIndex = Math.floor(exactIndex); + const rightIndex = Math.min(peakData.length - 1, leftIndex + 1); + const mix = exactIndex - leftIndex; + const amplitude = peakData[leftIndex] * (1 - mix) + peakData[rightIndex] * mix; const barHeight = amplitude * midY * 0.85; ctx.moveTo(px, midY - barHeight); diff --git a/src/components/video-editor/timeline/components/waveform/WaveformGenerator.ts b/src/components/video-editor/timeline/components/waveform/WaveformGenerator.ts new file mode 100644 index 00000000..7ff7f26c --- /dev/null +++ b/src/components/video-editor/timeline/components/waveform/WaveformGenerator.ts @@ -0,0 +1,88 @@ +import WorkerConstructor from "./waveform.worker?worker"; +import type { AudioPeaksData } from "../../core/timelineTypes"; + +const DEFAULT_PEAK_COUNT = 2048; + +export class WaveformGenerator { + private audioContext: AudioContext; + private worker: Worker; + private peaksCache = new Map(); + private pending = new Map>(); + + constructor() { + this.audioContext = new (window.AudioContext || (window as typeof window & { webkitAudioContext?: typeof AudioContext }).webkitAudioContext)(); + this.worker = new WorkerConstructor(); + } + + private computePeaksWithWorker(channelData: Float32Array, samples: number): Promise { + return new Promise((resolve, reject) => { + const onMessage = (event: MessageEvent) => { + this.worker.removeEventListener("message", onMessage); + this.worker.removeEventListener("error", onError); + resolve(event.data); + }; + const onError = (error: ErrorEvent) => { + this.worker.removeEventListener("message", onMessage); + this.worker.removeEventListener("error", onError); + reject(error.error ?? new Error(error.message)); + }; + + this.worker.addEventListener("message", onMessage); + this.worker.addEventListener("error", onError); + this.worker.postMessage( + { + channelData, + samples, + }, + [channelData.buffer], + ); + }); + } + + public async generate(url: string, peakCount = DEFAULT_PEAK_COUNT): Promise { + const cacheKey = `${url}::${peakCount}`; + const cached = this.peaksCache.get(cacheKey); + if (cached) return cached; + + const inflight = this.pending.get(cacheKey); + if (inflight) return inflight; + + const request = (async () => { + const response = await fetch(url); + if (!response.ok) { + throw new Error(`Failed to load media: ${response.status}`); + } + + const arrayBuffer = await response.arrayBuffer(); + const decoded = await this.audioContext.decodeAudioData(arrayBuffer); + const channelData = decoded.getChannelData(0).slice(); + const peaks = await this.computePeaksWithWorker(channelData, peakCount); + + let max = 0; + for (let i = 0; i < peaks.length; i++) { + if (peaks[i] > max) max = peaks[i]; + } + if (max > 0) { + for (let i = 0; i < peaks.length; i++) { + peaks[i] /= max; + } + } + + const result: AudioPeaksData = { + peaks, + durationMs: decoded.duration * 1000, + }; + this.peaksCache.set(cacheKey, result); + this.pending.delete(cacheKey); + return result; + })().catch((error) => { + this.pending.delete(cacheKey); + throw error; + }); + + this.pending.set(cacheKey, request); + return request; + } +} + +export const waveformGenerator = new WaveformGenerator(); diff --git a/src/components/video-editor/timeline/components/waveform/waveform.worker.ts b/src/components/video-editor/timeline/components/waveform/waveform.worker.ts new file mode 100644 index 00000000..c8a7f8c5 --- /dev/null +++ b/src/components/video-editor/timeline/components/waveform/waveform.worker.ts @@ -0,0 +1,31 @@ +self.onmessage = (e: MessageEvent) => { + const { channelData, samples } = e.data as { + channelData: Float32Array; + samples: number; + }; + + if (!channelData || samples <= 0) { + self.postMessage(new Float32Array(0)); + return; + } + + try { + const step = Math.max(1, Math.floor(channelData.length / samples)); + const result = new Float32Array(samples); + + for (let i = 0; i < samples; i++) { + const start = i * step; + const end = Math.min(start + step, channelData.length); + let max = 0; + for (let j = start; j < end; j++) { + const val = Math.abs(channelData[j]); + if (val > max) max = val; + } + result[i] = max; + } + + self.postMessage(result); + } catch { + self.postMessage(new Float32Array(0)); + } +}; diff --git a/src/components/video-editor/timeline/core/timelineTypes.ts b/src/components/video-editor/timeline/core/timelineTypes.ts index 2222f7cc..4a58c969 100644 --- a/src/components/video-editor/timeline/core/timelineTypes.ts +++ b/src/components/video-editor/timeline/core/timelineTypes.ts @@ -32,6 +32,7 @@ export interface TimelineRenderItem { rowId: string; span: Span; label: string; + audioPath?: string; zoomDepth?: number; zoomMode?: ZoomMode; speedValue?: number; diff --git a/src/components/video-editor/timeline/hooks/useTimelineAudioPeaks.ts b/src/components/video-editor/timeline/hooks/useTimelineAudioPeaks.ts index 9af79a80..4ca2a8f4 100644 --- a/src/components/video-editor/timeline/hooks/useTimelineAudioPeaks.ts +++ b/src/components/video-editor/timeline/hooks/useTimelineAudioPeaks.ts @@ -1,10 +1,9 @@ import { useEffect, useRef, useState } from "react"; -import { fromFileUrl, resolveVideoUrl } from "../../projectPersistence"; +import { resolveMediaResourceUrl } from "@/lib/exporter/localMediaSource"; +import { fromFileUrl } from "../../projectPersistence"; +import { waveformGenerator } from "../components/waveform/WaveformGenerator"; import type { AudioPeaksData } from "../core/timelineTypes"; -/** Number of peak bins to produce — enough for smooth display at any zoom. */ -const TARGET_PEAK_COUNT = 2048; - function buildSidecarAudioCandidates(sourcePath: string): string[] { const normalized = sourcePath.replace(/\\/g, "/"); const lastSlash = normalized.lastIndexOf("/"); @@ -29,193 +28,69 @@ function extractLocalPathFromMediaServerUrl(input: string): string | null { (url.hostname === "127.0.0.1" || url.hostname === "localhost") && url.pathname === "/video"; if (!isLocalMediaServer) return null; - const rawPath = url.searchParams.get("path"); - if (!rawPath) return null; - return rawPath; + return url.searchParams.get("path"); } catch { return null; } } -async function decodePeaksFromMediaUrl( - mediaUrl: string, - cancelled: () => boolean, -): Promise { - const response = await fetch(mediaUrl); - if (!response.ok) { - throw new Error(`Failed to load media for waveform: ${response.status}`); - } - if (cancelled()) throw new Error("cancelled"); - console.debug("[timeline-audio-peaks] fetch ok", { - url: mediaUrl, - status: response.status, - }); - - const arrayBuffer = await response.arrayBuffer(); - if (cancelled()) throw new Error("cancelled"); - console.debug("[timeline-audio-peaks] bytes loaded", { - url: mediaUrl, - byteLength: arrayBuffer.byteLength, - }); - - const audioCtx = new OfflineAudioContext(1, 1, 44100); - const decoded = await audioCtx.decodeAudioData(arrayBuffer); - if (cancelled()) throw new Error("cancelled"); - console.debug("[timeline-audio-peaks] decode ok", { - url: mediaUrl, - durationSec: decoded.duration, - channelCount: decoded.numberOfChannels, - sampleRate: decoded.sampleRate, - sampleFrames: decoded.length, - }); - - const channelData = decoded.getChannelData(0); - const durationMs = decoded.duration * 1000; - const binSize = Math.max(1, Math.floor(channelData.length / TARGET_PEAK_COUNT)); - const peakCount = Math.ceil(channelData.length / binSize); - const peaks = new Float32Array(peakCount); - - for (let i = 0; i < peakCount; i++) { - const start = i * binSize; - const end = Math.min(start + binSize, channelData.length); - let max = 0; - for (let j = start; j < end; j++) { - const abs = Math.abs(channelData[j]); - if (abs > max) max = abs; - } - peaks[i] = max; - } - - let globalMax = 0; - for (let i = 0; i < peaks.length; i++) { - if (peaks[i] > globalMax) globalMax = peaks[i]; - } - if (globalMax > 0) { - for (let i = 0; i < peaks.length; i++) { - peaks[i] /= globalMax; - } - } - - return { peaks, durationMs }; +interface TimelineAudioPeaksOptions { + enableSourceSidecarFallback?: boolean; } -/** - * Decode audio from a media file URL and produce a fixed-length array of peak - * amplitudes suitable for waveform visualisation. - * - * Returns `null` while loading or if the file has no decodeable audio. - */ -export function useTimelineAudioPeaks(fileUrl: string | null | undefined): AudioPeaksData | null { +export function useTimelineAudioPeaks( + mediaResource: string | null | undefined, + options: TimelineAudioPeaksOptions = {}, +): AudioPeaksData | null { const [data, setData] = useState(null); - const urlRef = useRef(fileUrl); + const sourceRef = useRef(mediaResource); + const enableSourceSidecarFallback = options.enableSourceSidecarFallback ?? false; useEffect(() => { - urlRef.current = fileUrl; + sourceRef.current = mediaResource; setData(null); - console.debug("[timeline-audio-peaks] reset", { input: fileUrl ?? null }); - - if (!fileUrl) { - console.debug("[timeline-audio-peaks] no input source, skipping"); - return; - } + if (!mediaResource) return; let cancelled = false; - (async () => { + const run = async () => { + const localPathFromServer = extractLocalPathFromMediaServerUrl(mediaResource); + const localSourcePath = + localPathFromServer || + (/^file:\/\//i.test(mediaResource) ? fromFileUrl(mediaResource) : mediaResource); + + const tryGenerate = async (resource: string): Promise => { + const resolvedUrl = await resolveMediaResourceUrl(resource); + return waveformGenerator.generate(resolvedUrl); + }; + try { - const extractedLocalPath = extractLocalPathFromMediaServerUrl(fileUrl); - const isRemoteLike = /^(https?:|blob:|data:)/i.test(fileUrl) && !extractedLocalPath; - const isFileUrl = /^file:\/\//i.test(fileUrl); - const sourceLocalPath = isRemoteLike - ? null - : extractedLocalPath - ? extractedLocalPath - : isFileUrl - ? fromFileUrl(fileUrl) - : fileUrl; - console.debug("[timeline-audio-peaks] resolving source", { - input: fileUrl, - isRemoteLike, - isFileUrl, - sourceLocalPath, - extractedLocalPath, - }); - const mediaUrl = isRemoteLike - ? fileUrl - : isFileUrl - ? await resolveVideoUrl(fromFileUrl(fileUrl)) - : await resolveVideoUrl(fileUrl); - if (cancelled) return; - console.debug("[timeline-audio-peaks] resolved source", { - input: fileUrl, - resolved: mediaUrl, - }); - - let decodedPeaks: AudioPeaksData | null = null; - try { - decodedPeaks = await decodePeaksFromMediaUrl(mediaUrl, () => cancelled); - } catch (primaryError) { - if (primaryError instanceof Error && primaryError.message === "cancelled") { - return; - } - console.warn("[timeline-audio-peaks] primary decode failed", { - input: fileUrl, - url: mediaUrl, - error: primaryError, - }); - if (sourceLocalPath) { - const candidates = buildSidecarAudioCandidates(sourceLocalPath); - console.debug("[timeline-audio-peaks] trying sidecar candidates", { - sourceLocalPath, - candidates, - }); - for (const candidatePath of candidates) { - try { - const candidateUrl = await resolveVideoUrl(candidatePath); - console.debug("[timeline-audio-peaks] trying sidecar", { - candidatePath, - candidateUrl, - }); - decodedPeaks = await decodePeaksFromMediaUrl(candidateUrl, () => cancelled); - console.info("[timeline-audio-peaks] sidecar decode succeeded", { - candidatePath, - candidateUrl, - }); - break; - } catch (sidecarError) { - if (sidecarError instanceof Error && sidecarError.message === "cancelled") { - return; - } - console.debug("[timeline-audio-peaks] sidecar decode failed", { - candidatePath, - error: sidecarError, - }); - } - } - } - } - - if (!cancelled && urlRef.current === fileUrl && decodedPeaks) { - console.debug("[timeline-audio-peaks] peaks generated", { - input: fileUrl, - durationMs: Math.round(decodedPeaks.durationMs), - peakCount: decodedPeaks.peaks.length, - }); - setData(decodedPeaks); - } - } catch (error) { - // File has no audio or decoding failed — leave as null. - console.warn("[timeline-audio-peaks] failed", { - input: fileUrl, - error, - }); + const result = await tryGenerate(mediaResource); + if (!cancelled && sourceRef.current === mediaResource) setData(result); + return; + } catch { + // fallthrough } - })(); + + if (!enableSourceSidecarFallback || !localSourcePath) return; + + for (const candidate of buildSidecarAudioCandidates(localSourcePath)) { + try { + const result = await tryGenerate(candidate); + if (!cancelled && sourceRef.current === mediaResource) setData(result); + return; + } catch { + // try next + } + } + }; + + void run(); return () => { cancelled = true; }; - }, [fileUrl]); + }, [mediaResource, enableSourceSidecarFallback]); return data; } diff --git a/src/components/video-editor/timeline/model/timelineModel.ts b/src/components/video-editor/timeline/model/timelineModel.ts index 0981b85c..b36aabc9 100644 --- a/src/components/video-editor/timeline/model/timelineModel.ts +++ b/src/components/video-editor/timeline/model/timelineModel.ts @@ -68,6 +68,7 @@ export function buildTimelineItems(params: { rowId: getAudioTrackRowId(region.trackIndex ?? 0), span: { start: region.startMs, end: region.endMs }, label: getAudioLabel(region), + audioPath: region.audioPath, variant: "audio", })); From bfb0e821659dff1851c0a20a36a349bd8a0df44d Mon Sep 17 00:00:00 2001 From: Alan Trebugeais Date: Sat, 9 May 2026 13:38:04 +0200 Subject: [PATCH 03/25] fix waveform possible problems --- .../components/waveform/WaveformGenerator.ts | 29 +++++++++++++------ .../components/waveform/waveform.worker.ts | 9 +++--- 2 files changed, 25 insertions(+), 13 deletions(-) diff --git a/src/components/video-editor/timeline/components/waveform/WaveformGenerator.ts b/src/components/video-editor/timeline/components/waveform/WaveformGenerator.ts index 7ff7f26c..07d00419 100644 --- a/src/components/video-editor/timeline/components/waveform/WaveformGenerator.ts +++ b/src/components/video-editor/timeline/components/waveform/WaveformGenerator.ts @@ -8,29 +8,40 @@ export class WaveformGenerator { private worker: Worker; private peaksCache = new Map(); private pending = new Map>(); + private workerRequestSeq = 0; + private workerResolvers = new Map void>(); constructor() { this.audioContext = new (window.AudioContext || (window as typeof window & { webkitAudioContext?: typeof AudioContext }).webkitAudioContext)(); this.worker = new WorkerConstructor(); + this.worker.addEventListener( + "message", + (event: MessageEvent<{ requestId: number; peaks: Float32Array }>) => { + const { requestId, peaks } = event.data; + const resolve = this.workerResolvers.get(requestId); + if (!resolve) return; + this.workerResolvers.delete(requestId); + resolve(peaks); + }, + ); } private computePeaksWithWorker(channelData: Float32Array, samples: number): Promise { return new Promise((resolve, reject) => { - const onMessage = (event: MessageEvent) => { - this.worker.removeEventListener("message", onMessage); - this.worker.removeEventListener("error", onError); - resolve(event.data); - }; + const requestId = ++this.workerRequestSeq; const onError = (error: ErrorEvent) => { - this.worker.removeEventListener("message", onMessage); this.worker.removeEventListener("error", onError); + this.workerResolvers.delete(requestId); reject(error.error ?? new Error(error.message)); }; - - this.worker.addEventListener("message", onMessage); - this.worker.addEventListener("error", onError); + this.worker.addEventListener("error", onError, { once: true }); + this.workerResolvers.set(requestId, (peaks) => { + this.worker.removeEventListener("error", onError); + resolve(peaks); + }); this.worker.postMessage( { + requestId, channelData, samples, }, diff --git a/src/components/video-editor/timeline/components/waveform/waveform.worker.ts b/src/components/video-editor/timeline/components/waveform/waveform.worker.ts index c8a7f8c5..7b7a5184 100644 --- a/src/components/video-editor/timeline/components/waveform/waveform.worker.ts +++ b/src/components/video-editor/timeline/components/waveform/waveform.worker.ts @@ -1,11 +1,12 @@ self.onmessage = (e: MessageEvent) => { - const { channelData, samples } = e.data as { + const { requestId, channelData, samples } = e.data as { + requestId: number; channelData: Float32Array; samples: number; }; if (!channelData || samples <= 0) { - self.postMessage(new Float32Array(0)); + self.postMessage({ requestId, peaks: new Float32Array(0) }); return; } @@ -24,8 +25,8 @@ self.onmessage = (e: MessageEvent) => { result[i] = max; } - self.postMessage(result); + self.postMessage({ requestId, peaks: result }); } catch { - self.postMessage(new Float32Array(0)); + self.postMessage({ requestId, peaks: new Float32Array(0) }); } }; From 26665415854b665a680e12ad26116b6bdd8b9281 Mon Sep 17 00:00:00 2001 From: Alan Trebugeais Date: Sat, 9 May 2026 13:46:10 +0200 Subject: [PATCH 04/25] fix code quality, and also isue with audio not being the right spot --- src/components/video-editor/timeline/Item.tsx | 3 + .../components/viewport/TimelineCanvas.tsx | 10 ++- .../components/waveform/AudioWaveform.tsx | 90 ++++++++++--------- .../components/waveform/WaveformGenerator.ts | 5 +- .../components/wrapper/TimelineWrapper.tsx | 8 +- .../video-editor/timeline/core/constants.ts | 2 + .../timeline/hooks/useTimelineAudioPeaks.ts | 20 +++-- 7 files changed, 81 insertions(+), 57 deletions(-) diff --git a/src/components/video-editor/timeline/Item.tsx b/src/components/video-editor/timeline/Item.tsx index 33703a82..e633e52e 100644 --- a/src/components/video-editor/timeline/Item.tsx +++ b/src/components/video-editor/timeline/Item.tsx @@ -19,6 +19,7 @@ interface ItemProps { id: string; span: Span; rowId: string; + disabled?: boolean; children: React.ReactNode; isSelected?: boolean; onSelect?: () => void; @@ -55,6 +56,7 @@ export default function Item({ id, span, rowId, + disabled = false, isSelected = false, onSelect, onSelectId, @@ -69,6 +71,7 @@ export default function Item({ const { setNodeRef, attributes, listeners, itemStyle, itemContentStyle } = useItem({ id, span, + disabled, data: { rowId }, }); diff --git a/src/components/video-editor/timeline/components/viewport/TimelineCanvas.tsx b/src/components/video-editor/timeline/components/viewport/TimelineCanvas.tsx index c7d65898..d0dae8c2 100644 --- a/src/components/video-editor/timeline/components/viewport/TimelineCanvas.tsx +++ b/src/components/video-editor/timeline/components/viewport/TimelineCanvas.tsx @@ -20,7 +20,7 @@ import { import glassStyles from "../../ItemGlass.module.css"; import Item from "../../Item"; import Row from "../../Row"; -import { CLIP_ROW_ID, ZOOM_ROW_ID } from "../../core/constants"; +import { CLIP_ROW_ID, SOURCE_AUDIO_ROW_ID, ZOOM_ROW_ID } from "../../core/constants"; import type { AudioPeaksData, TimelineRenderItem } from "../../core/timelineTypes"; import { getAnnotationTrackIndex, @@ -38,7 +38,6 @@ import { useTimelineAudioPeaks } from "../../hooks/useTimelineAudioPeaks"; const HINT_CLIP = "Press C to split clip"; const HINT_ANNOTATION = "Press A to add annotation"; const HINT_AUDIO = "Click music icon to add audio"; -const SOURCE_AUDIO_ROW_ID = "row-source-audio"; interface TimelineCanvasProps { items: TimelineRenderItem[]; @@ -251,6 +250,10 @@ function AudioItemWithWaveform({ onSelectAudio, }: AudioItemWithWaveformProps) { const peaks = useTimelineAudioPeaks(item.audioPath ?? null); + const normalizedWaveformSpan = useMemo(() => { + const duration = Math.max(0, waveformSpan.end - waveformSpan.start); + return { start: 0, end: duration }; + }, [waveformSpan.end, waveformSpan.start]); return ( {item.label} @@ -370,6 +373,7 @@ const TimelineCanvasRows = memo(function TimelineCanvasRows({ id={`source-audio-${item.id}`} rowId={SOURCE_AUDIO_ROW_ID} span={liveSpanPreviewById?.[item.id] ?? item.span} + disabled isSelected={selectAllBlocksActive || item.id === selectedClipId} onSelect={() => onSelectClip?.(item.id)} variant="audio" diff --git a/src/components/video-editor/timeline/components/waveform/AudioWaveform.tsx b/src/components/video-editor/timeline/components/waveform/AudioWaveform.tsx index eff26d09..4b8bf510 100644 --- a/src/components/video-editor/timeline/components/waveform/AudioWaveform.tsx +++ b/src/components/video-editor/timeline/components/waveform/AudioWaveform.tsx @@ -23,6 +23,7 @@ function AudioWaveformComponent({ const canvasRef = useRef(null); const { range } = useTimelineContext(); const [resizeKey, setResizeKey] = useState(0); + const lastDrawAtRef = useRef(0); // Bump resizeKey when the canvas element changes size. const observerRef = useRef(null); @@ -42,57 +43,66 @@ function AudioWaveformComponent({ useEffect(() => { const canvas = canvasRef.current; if (!canvas) return; + let rafId = 0; - const ctx = canvas.getContext("2d"); - if (!ctx) return; + const draw = () => { + const now = performance.now(); + if (now - lastDrawAtRef.current < 33) return; + lastDrawAtRef.current = now; - const rect = canvas.getBoundingClientRect(); - const dpr = window.devicePixelRatio || 1; - const width = Math.round(rect.width * dpr); - const height = Math.round(rect.height * dpr); + const ctx = canvas.getContext("2d"); + if (!ctx) return; - if (width === 0 || height === 0) return; + const rect = canvas.getBoundingClientRect(); + const dpr = window.devicePixelRatio || 1; + const width = Math.round(rect.width * dpr); + const height = Math.round(rect.height * dpr); - canvas.width = width; - canvas.height = height; + if (width === 0 || height === 0) return; - ctx.clearRect(0, 0, width, height); + canvas.width = width; + canvas.height = height; - const { peaks: peakData, durationMs } = peaks; - if (durationMs <= 0 || peakData.length === 0) return; + ctx.clearRect(0, 0, width, height); - const rawVisibleStartMs = segmentStartMs ?? range.start; - const rawVisibleEndMs = segmentEndMs ?? range.end; - const msPerBin = durationMs / peakData.length; - const visibleStartMs = - msPerBin > 0 ? Math.round(rawVisibleStartMs / msPerBin) * msPerBin : rawVisibleStartMs; - const visibleEndMs = - msPerBin > 0 ? Math.round(rawVisibleEndMs / msPerBin) * msPerBin : rawVisibleEndMs; - const visibleDurationMs = visibleEndMs - visibleStartMs; - if (visibleDurationMs <= 0) return; + const { peaks: peakData, durationMs } = peaks; + if (durationMs <= 0 || peakData.length === 0) return; - const midY = height / 2; + const rawVisibleStartMs = segmentStartMs ?? range.start; + const rawVisibleEndMs = segmentEndMs ?? range.end; + const msPerBin = durationMs / peakData.length; + const visibleStartMs = + msPerBin > 0 ? Math.round(rawVisibleStartMs / msPerBin) * msPerBin : rawVisibleStartMs; + const visibleEndMs = + msPerBin > 0 ? Math.round(rawVisibleEndMs / msPerBin) * msPerBin : rawVisibleEndMs; + const visibleDurationMs = visibleEndMs - visibleStartMs; + if (visibleDurationMs <= 0) return; - ctx.beginPath(); - for (let px = 0; px < width; px++) { - const t = visibleStartMs + (px / width) * visibleDurationMs; - const exactIndex = Math.max( - 0, - Math.min(peakData.length - 1, (t / durationMs) * (peakData.length - 1)), - ); - const leftIndex = Math.floor(exactIndex); - const rightIndex = Math.min(peakData.length - 1, leftIndex + 1); - const mix = exactIndex - leftIndex; - const amplitude = peakData[leftIndex] * (1 - mix) + peakData[rightIndex] * mix; - const barHeight = amplitude * midY * 0.85; + const midY = height / 2; - ctx.moveTo(px, midY - barHeight); - ctx.lineTo(px, midY + barHeight); - } + ctx.beginPath(); + for (let px = 0; px < width; px++) { + const t = visibleStartMs + (px / width) * visibleDurationMs; + const exactIndex = Math.max( + 0, + Math.min(peakData.length - 1, (t / durationMs) * (peakData.length - 1)), + ); + const leftIndex = Math.floor(exactIndex); + const rightIndex = Math.min(peakData.length - 1, leftIndex + 1); + const mix = exactIndex - leftIndex; + const amplitude = peakData[leftIndex] * (1 - mix) + peakData[rightIndex] * mix; + const barHeight = amplitude * midY * 0.85; - ctx.strokeStyle = "rgba(255, 255, 255, 0.55)"; - ctx.lineWidth = dpr; - ctx.stroke(); + ctx.moveTo(px, midY - barHeight); + ctx.lineTo(px, midY + barHeight); + } + + ctx.strokeStyle = "rgba(255, 255, 255, 0.55)"; + ctx.lineWidth = dpr; + ctx.stroke(); + }; + rafId = requestAnimationFrame(draw); + return () => cancelAnimationFrame(rafId); }, [peaks, range.start, range.end, resizeKey, segmentStartMs, segmentEndMs]); return ( diff --git a/src/components/video-editor/timeline/components/waveform/WaveformGenerator.ts b/src/components/video-editor/timeline/components/waveform/WaveformGenerator.ts index 07d00419..128a65ad 100644 --- a/src/components/video-editor/timeline/components/waveform/WaveformGenerator.ts +++ b/src/components/video-editor/timeline/components/waveform/WaveformGenerator.ts @@ -1,7 +1,6 @@ import WorkerConstructor from "./waveform.worker?worker"; import type { AudioPeaksData } from "../../core/timelineTypes"; - -const DEFAULT_PEAK_COUNT = 2048; +import { WAVEFORM_DEFAULT_PEAK_COUNT } from "../../core/constants"; export class WaveformGenerator { private audioContext: AudioContext; @@ -50,7 +49,7 @@ export class WaveformGenerator { }); } - public async generate(url: string, peakCount = DEFAULT_PEAK_COUNT): Promise { + public async generate(url: string, peakCount = WAVEFORM_DEFAULT_PEAK_COUNT): Promise { const cacheKey = `${url}::${peakCount}`; const cached = this.peaksCache.get(cacheKey); if (cached) return cached; diff --git a/src/components/video-editor/timeline/components/wrapper/TimelineWrapper.tsx b/src/components/video-editor/timeline/components/wrapper/TimelineWrapper.tsx index 917b171a..db0925c1 100644 --- a/src/components/video-editor/timeline/components/wrapper/TimelineWrapper.tsx +++ b/src/components/video-editor/timeline/components/wrapper/TimelineWrapper.tsx @@ -134,9 +134,8 @@ export default function TimelineWrapper({ (event: DragStartEvent) => { const span = event.active.data.current.getSpanFromDragEvent?.(event); if (span) showTooltip(span); - onLiveSpanPreviewChange?.(event.active.id as string, span ?? null); }, - [onLiveSpanPreviewChange, showTooltip], + [showTooltip], ); const onDragMove = useCallback( @@ -147,7 +146,10 @@ export default function TimelineWrapper({ ? (event.activatorEvent as PointerEvent).clientX + (event.delta?.x ?? 0) : undefined; if (span) showTooltip(span, screenX); - onLiveSpanPreviewChange?.(event.active.id as string, span ?? null); + const moved = Math.abs(event.delta?.x ?? 0) > 0.01; + if (moved) { + onLiveSpanPreviewChange?.(event.active.id as string, span ?? null); + } }, [onLiveSpanPreviewChange, showTooltip], ); diff --git a/src/components/video-editor/timeline/core/constants.ts b/src/components/video-editor/timeline/core/constants.ts index 1b71e996..88acba23 100644 --- a/src/components/video-editor/timeline/core/constants.ts +++ b/src/components/video-editor/timeline/core/constants.ts @@ -2,8 +2,10 @@ export const ZOOM_ROW_ID = "row-zoom"; export const CLIP_ROW_ID = "row-clip"; export const ANNOTATION_ROW_ID = "row-annotation"; export const AUDIO_ROW_ID = "row-audio"; +export const SOURCE_AUDIO_ROW_ID = "row-source-audio"; export const ANNOTATION_ROW_PREFIX = `${ANNOTATION_ROW_ID}-`; export const AUDIO_ROW_PREFIX = `${AUDIO_ROW_ID}-`; export const FALLBACK_RANGE_MS = 1000; export const TARGET_MARKER_COUNT = 12; +export const WAVEFORM_DEFAULT_PEAK_COUNT = 2048; diff --git a/src/components/video-editor/timeline/hooks/useTimelineAudioPeaks.ts b/src/components/video-editor/timeline/hooks/useTimelineAudioPeaks.ts index 4ca2a8f4..499bab2b 100644 --- a/src/components/video-editor/timeline/hooks/useTimelineAudioPeaks.ts +++ b/src/components/video-editor/timeline/hooks/useTimelineAudioPeaks.ts @@ -2,6 +2,7 @@ import { useEffect, useRef, useState } from "react"; import { resolveMediaResourceUrl } from "@/lib/exporter/localMediaSource"; import { fromFileUrl } from "../../projectPersistence"; import { waveformGenerator } from "../components/waveform/WaveformGenerator"; +import { WAVEFORM_DEFAULT_PEAK_COUNT } from "../core/constants"; import type { AudioPeaksData } from "../core/timelineTypes"; function buildSidecarAudioCandidates(sourcePath: string): string[] { @@ -36,6 +37,7 @@ function extractLocalPathFromMediaServerUrl(input: string): string | null { interface TimelineAudioPeaksOptions { enableSourceSidecarFallback?: boolean; + peakCount?: number; } export function useTimelineAudioPeaks( @@ -45,6 +47,7 @@ export function useTimelineAudioPeaks( const [data, setData] = useState(null); const sourceRef = useRef(mediaResource); const enableSourceSidecarFallback = options.enableSourceSidecarFallback ?? false; + const peakCount = options.peakCount ?? WAVEFORM_DEFAULT_PEAK_COUNT; useEffect(() => { sourceRef.current = mediaResource; @@ -54,14 +57,9 @@ export function useTimelineAudioPeaks( let cancelled = false; const run = async () => { - const localPathFromServer = extractLocalPathFromMediaServerUrl(mediaResource); - const localSourcePath = - localPathFromServer || - (/^file:\/\//i.test(mediaResource) ? fromFileUrl(mediaResource) : mediaResource); - const tryGenerate = async (resource: string): Promise => { const resolvedUrl = await resolveMediaResourceUrl(resource); - return waveformGenerator.generate(resolvedUrl); + return waveformGenerator.generate(resolvedUrl, peakCount); }; try { @@ -72,7 +70,13 @@ export function useTimelineAudioPeaks( // fallthrough } - if (!enableSourceSidecarFallback || !localSourcePath) return; + if (!enableSourceSidecarFallback) return; + + const localPathFromServer = extractLocalPathFromMediaServerUrl(mediaResource); + const localSourcePath = + localPathFromServer || + (/^file:\/\//i.test(mediaResource) ? fromFileUrl(mediaResource) : mediaResource); + if (!localSourcePath) return; for (const candidate of buildSidecarAudioCandidates(localSourcePath)) { try { @@ -90,7 +94,7 @@ export function useTimelineAudioPeaks( return () => { cancelled = true; }; - }, [mediaResource, enableSourceSidecarFallback]); + }, [mediaResource, enableSourceSidecarFallback, peakCount]); return data; } From 1c7bcffd25cb30488e1f31a9d4420cb31426e436 Mon Sep 17 00:00:00 2001 From: Alan Trebugeais Date: Sat, 9 May 2026 14:30:35 +0200 Subject: [PATCH 05/25] refactor : video-editor/audio to put all the timeline and video editor logic about audio here. reducing further video editor --- src/components/video-editor/SettingsPanel.tsx | 108 ++++- src/components/video-editor/VideoEditor.tsx | 437 ++++-------------- .../video-editor/audio/clipAudio.ts | 19 + .../video-editor/audio/sourceAudioTracks.ts | 9 + .../video-editor/audio/useAudioPreviewSync.ts | 322 +++++++++++++ .../audio/useSourceAudioFallback.ts | 68 +++ .../audio/useSourceAudioTrackSettings.ts | 112 +++++ .../waveform/WaveformGenerator.ts | 4 +- .../waveform/waveform.worker.ts | 0 .../video-editor/timeline/TimelineEditor.tsx | 96 +++- .../components/viewport/TimelineCanvas.tsx | 60 +-- .../timeline/hooks/useTimelineAudioPeaks.ts | 2 +- 12 files changed, 851 insertions(+), 386 deletions(-) create mode 100644 src/components/video-editor/audio/clipAudio.ts create mode 100644 src/components/video-editor/audio/sourceAudioTracks.ts create mode 100644 src/components/video-editor/audio/useAudioPreviewSync.ts create mode 100644 src/components/video-editor/audio/useSourceAudioFallback.ts create mode 100644 src/components/video-editor/audio/useSourceAudioTrackSettings.ts rename src/components/video-editor/{timeline/components => audio}/waveform/WaveformGenerator.ts (95%) rename src/components/video-editor/{timeline/components => audio}/waveform/waveform.worker.ts (100%) diff --git a/src/components/video-editor/SettingsPanel.tsx b/src/components/video-editor/SettingsPanel.tsx index 71c9ce53..657398a2 100644 --- a/src/components/video-editor/SettingsPanel.tsx +++ b/src/components/video-editor/SettingsPanel.tsx @@ -469,8 +469,15 @@ interface SettingsPanelProps { selectedClipId?: string | null; selectedClipSpeed?: number | null; selectedClipMuted?: boolean | null; + hasClipSourceAudio?: boolean; + showClipSourceAudioTrack?: boolean; onClipSpeedChange?: (speed: number) => void; onClipMutedChange?: (muted: boolean) => void; + onShowClipSourceAudioTrackChange?: (show: boolean) => void; + sourceAudioTrackMeta?: Array<{ id: string; label: string }>; + sourceAudioTrackSettings?: Record; + onSourceAudioTrackVolumeChange?: (id: string, volume: number) => void; + onSourceAudioTrackNormalizeChange?: (id: string, normalize: boolean) => void; onClipDelete?: (id: string) => void; selectedAudioId?: string | null; selectedAudioVolume?: number | null; @@ -862,8 +869,15 @@ export function SettingsPanel({ selectedClipId, selectedClipSpeed, selectedClipMuted, + hasClipSourceAudio = false, + showClipSourceAudioTrack = false, onClipSpeedChange, onClipMutedChange, + onShowClipSourceAudioTrackChange, + sourceAudioTrackMeta = [], + sourceAudioTrackSettings = {}, + onSourceAudioTrackVolumeChange, + onSourceAudioTrackNormalizeChange, onClipDelete, selectedAudioId, selectedAudioVolume, @@ -3049,12 +3063,38 @@ export function SettingsPanel({ {tSettings("clip.muteAudio", "Mute Audio")} - onClipMutedChange?.(v)} - className="data-[state=checked]:bg-[#06b6d4] scale-75" - /> +
+ + {selectedClipMuted + ? tSettings("clip.mutedState", "Muted") + : tSettings("clip.audioOnState", "Audio On")} + + onClipMutedChange?.(!v)} + className="data-[state=checked]:bg-[#06b6d4] scale-75" + /> +
+ {hasClipSourceAudio && ( +
+ + {tSettings("clip.separateSourceAudio", "Separate Clip Audio")} + + onShowClipSourceAudioTrackChange?.(v)} + className="data-[state=checked]:bg-[#06b6d4] scale-75" + /> +
+ )}
{tSettings("speed.label", "Speed")}
@@ -3562,10 +3602,12 @@ export function SettingsPanel({
- {selectedAudioId && ( + {selectedAudioId ? (
@@ -3596,7 +3638,57 @@ export function SettingsPanel({ {tSettings("audio.deleteRegion", "Delete Audio")}
- )} + ) : selectedClipId && hasClipSourceAudio && showClipSourceAudioTrack ? ( +
+
+ {tSettings("audio.sourceTracksTitle", "Clip Source Audio")} +
+ {sourceAudioTrackMeta.map((track) => { + const settings = sourceAudioTrackSettings[track.id] ?? { + volume: 1, + normalize: false, + }; + return ( +
+
+ + {track.label} + + + {Math.round(settings.volume * 100)}% + +
+
+ + {tSettings("audio.normalize", "Normalize")} + + + onSourceAudioTrackNormalizeChange?.(track.id, v) + } + className="data-[state=checked]:bg-[#2563EB] scale-75" + /> +
+ onSourceAudioTrackVolumeChange?.(track.id, v)} + formatValue={(v) => `${Math.round(v * 100)}%`} + parseInput={(text) => parseFloat(text.replace(/%$/, "")) / 100} + /> +
+ ); + })} +
+ ) : null}
); diff --git a/src/components/video-editor/VideoEditor.tsx b/src/components/video-editor/VideoEditor.tsx index fa5fc5ac..6208ca3f 100644 --- a/src/components/video-editor/VideoEditor.tsx +++ b/src/components/video-editor/VideoEditor.tsx @@ -80,14 +80,7 @@ import { canUseInMemoryExportSaveFallback, describeBlockedInMemoryExportSave, } from "@/lib/exporter/exportSavePolicy"; -import { resolveMediaElementSource } from "@/lib/exporter/localMediaSource"; import { resolveSourceAudioFallbackPaths } from "@/lib/exporter/sourceAudioFallback"; -import { - clampMediaTimeToDuration, - enablePitchPreservingPlayback, - estimateCompanionAudioStartDelaySeconds, - getMediaSyncPlaybackRate, -} from "@/lib/mediaTiming"; import { matchesShortcut } from "@/lib/shortcuts"; import { cn } from "@/lib/utils"; import { @@ -147,6 +140,11 @@ import { validateProjectData, } from "./projectPersistence"; import { SettingsPanel } from "./SettingsPanel"; +import { SOURCE_AUDIO_NORMALIZE_GAIN, getSourceTrackIdFromPath } from "./audio/sourceAudioTracks"; +import { useAudioPreviewSync } from "./audio/useAudioPreviewSync"; +import { useSourceAudioFallback } from "./audio/useSourceAudioFallback"; +import { getActiveClipIdAtSourceTime, isClipMutedById } from "./audio/clipAudio"; +import { useSourceAudioTrackSettings } from "./audio/useSourceAudioTrackSettings"; import { APP_HEADER_ICON_BUTTON_CLASS, DiscordLinkButton, @@ -347,12 +345,6 @@ async function writeSmokeExportReport( const SMOKE_EXPORT_READY_TIMEOUT_MS = 30_000; const DEFAULT_MP4_EXPORT_FRAME_RATE: ExportMp4FrameRate = 30; -const SOURCE_AUDIO_FALLBACK_TOAST_ID = "source-audio-fallback-error"; -const SOURCE_AUDIO_PREVIEW_PLAYING_SEEK_DRIFT_SECONDS = 0.18; -const SOURCE_AUDIO_PREVIEW_PAUSED_SEEK_DRIFT_SECONDS = 0.01; -const SOURCE_AUDIO_PREVIEW_RATE_TOLERANCE_SECONDS = 0.08; -const SOURCE_AUDIO_PREVIEW_RATE_CORRECTION_WINDOW_SECONDS = 8; -const SOURCE_AUDIO_PREVIEW_MAX_RATE_ADJUSTMENT = 0.015; const PROJECT_AUTOSAVE_DELAY_MS = 1000; const EXPORT_ERROR_TOAST_DURATION_MS = 20000; @@ -677,6 +669,8 @@ export default function VideoEditor() { const [selectedAnnotationId, setSelectedAnnotationId] = useState(null); const [audioRegions, setAudioRegions] = useState([]); const [selectedAudioId, setSelectedAudioId] = useState(null); + const [hasClipSourceAudio, setHasClipSourceAudio] = useState(false); + const [showClipSourceAudioTrack, setShowClipSourceAudioTrack] = useState(false); const [autoCaptions, setAutoCaptions] = useState([]); const [autoCaptionSettings, setAutoCaptionSettings] = useState( DEFAULT_AUTO_CAPTION_SETTINGS, @@ -700,9 +694,6 @@ export default function VideoEditor() { const [exportError, setExportError] = useState(null); const [showExportDropdown, setShowExportDropdown] = useState(false); const [previewVolume, setPreviewVolume] = useState(1); - const [sourceAudioFallbackPaths, setSourceAudioFallbackPaths] = useState([]); - const [sourceAudioFallbackStartDelayMsByPath, setSourceAudioFallbackStartDelayMsByPath] = - useState>({}); const applySessionPresentation = useCallback( ( session: @@ -1778,62 +1769,49 @@ export default function VideoEditor() { () => videoSourcePath ?? (videoPath ? fromFileUrl(videoPath) : null), [videoPath, videoSourcePath], ); + const { sourceAudioFallbackPaths, sourceAudioFallbackStartDelayMsByPath } = + useSourceAudioFallback({ + currentSourcePath, + summarizeErrorMessage, + }); const { hasEmbeddedSourceAudio, externalAudioPaths: previewSourceAudioFallbackPaths } = useMemo( () => resolveSourceAudioFallbackPaths(currentSourcePath, sourceAudioFallbackPaths), [currentSourcePath, sourceAudioFallbackPaths], ); const shouldMutePreviewVideo = !hasEmbeddedSourceAudio && previewSourceAudioFallbackPaths.length > 0; - - useEffect(() => { - let cancelled = false; - setSourceAudioFallbackPaths([]); - setSourceAudioFallbackStartDelayMsByPath({}); - - if (!currentSourcePath) { - return () => { - cancelled = true; - }; - } - - void (async () => { - try { - const result = - await window.electronAPI.getVideoAudioFallbackPaths(currentSourcePath); - if (cancelled) { - return; - } - if (!result.success) { - setSourceAudioFallbackPaths([]); - setSourceAudioFallbackStartDelayMsByPath({}); - toast.warning( - result.error - ? `Could not load companion audio sources: ${summarizeErrorMessage(result.error)}` - : "Could not load companion audio sources. Playback and export may miss microphone audio.", - { id: SOURCE_AUDIO_FALLBACK_TOAST_ID, duration: 10000 }, - ); - return; - } - - toast.dismiss(SOURCE_AUDIO_FALLBACK_TOAST_ID); - setSourceAudioFallbackPaths(result.paths ?? []); - setSourceAudioFallbackStartDelayMsByPath(result.startDelayMsByPath ?? {}); - } catch (error) { - if (!cancelled) { - setSourceAudioFallbackPaths([]); - setSourceAudioFallbackStartDelayMsByPath({}); - toast.warning( - `Could not load companion audio sources: ${summarizeErrorMessage(String(error))}`, - { id: SOURCE_AUDIO_FALLBACK_TOAST_ID, duration: 10000 }, - ); - } - } - })(); - - return () => { - cancelled = true; - }; - }, [currentSourcePath]); + const activeClipIdAtCurrentTime = useMemo( + () => getActiveClipIdAtSourceTime(currentTime, clipRegions), + [clipRegions, currentTime], + ); + const { + sourceAudioTrackMeta, + activeSourceAudioTrackSettings, + selectedClipSourceAudioTrackSettings, + onSourceAudioTracksMetaChange, + onSelectedClipSourceAudioTrackVolumeChange, + onSelectedClipSourceAudioTrackNormalizeChange, + } = useSourceAudioTrackSettings({ + selectedClipId, + activeClipId: activeClipIdAtCurrentTime, + }); + const embeddedSourcePreviewGain = useMemo(() => { + const settings = activeSourceAudioTrackSettings.mixed ?? { volume: 1, normalize: false }; + const normalizeGain = settings.normalize ? SOURCE_AUDIO_NORMALIZE_GAIN : 1; + return Math.max(0, Math.min(2, settings.volume * normalizeGain)); + }, [activeSourceAudioTrackSettings]); + const getSourceTrackPreviewGain = useCallback( + (audioPath: string) => { + const trackId = getSourceTrackIdFromPath(audioPath); + const settings = activeSourceAudioTrackSettings[trackId] ?? { volume: 1, normalize: false }; + const normalizeGain = settings.normalize ? SOURCE_AUDIO_NORMALIZE_GAIN : 1; + return Math.max(0, Math.min(2, settings.volume * normalizeGain)); + }, + [activeSourceAudioTrackSettings], + ); + const isCurrentClipMuted = useMemo(() => { + return isClipMutedById(activeClipIdAtCurrentTime, clipRegions); + }, [activeClipIdAtCurrentTime, clipRegions]); const projectDisplayName = useMemo(() => { const fileName = @@ -4096,289 +4074,24 @@ export default function VideoEditor() { } }, [selectedAudioId, audioRegions]); - // Audio playback sync: manage Audio elements that play in sync with video - const audioElementsRef = useRef>(new Map()); - const audioElementRevokersRef = useRef void>>(new Map()); - const audioElementResourcesRef = useRef>(new Map()); - const sourceAudioElementsRef = useRef>(new Map()); - const sourceAudioElementRevokersRef = useRef void>>(new Map()); - const sourceAudioElementResourcesRef = useRef>(new Map()); - const lastSourceAudioSyncTimeRef = useRef(null); - - useEffect(() => { - let cancelled = false; - const existing = audioElementsRef.current; - const currentIds = new Set(audioRegions.map((r) => r.id)); - - // Remove old audio elements - for (const [id, audio] of existing) { - if (!currentIds.has(id)) { - audio.pause(); - audio.src = ""; - audioElementRevokersRef.current.get(id)?.(); - audioElementRevokersRef.current.delete(id); - audioElementResourcesRef.current.delete(id); - existing.delete(id); - } - } - - // Create/update audio elements - for (const region of audioRegions) { - let audio = existing.get(region.id); - if (!audio) { - audio = new Audio(); - audio.preload = "auto"; - existing.set(region.id, audio); - } - - if (audioElementResourcesRef.current.get(region.id) !== region.audioPath) { - audio.pause(); - audio.src = ""; - audioElementRevokersRef.current.get(region.id)?.(); - audioElementRevokersRef.current.delete(region.id); - audioElementResourcesRef.current.set(region.id, region.audioPath); - - void (async () => { - const resolved = await resolveMediaElementSource(region.audioPath); - const latestAudio = existing.get(region.id); - - if ( - cancelled || - latestAudio !== audio || - audioElementResourcesRef.current.get(region.id) !== region.audioPath - ) { - resolved.revoke(); - return; - } - - audioElementRevokersRef.current.set(region.id, resolved.revoke); - latestAudio.src = resolved.src; - })(); - } - - audio.volume = Math.max(0, Math.min(1, region.volume * previewVolume)); - } - - return () => { - cancelled = true; - }; - }, [audioRegions, previewVolume]); - - useEffect(() => { - let cancelled = false; - const existing = sourceAudioElementsRef.current; - const currentIds = new Set(previewSourceAudioFallbackPaths); - - for (const [id, audio] of existing) { - if (!currentIds.has(id)) { - audio.pause(); - audio.src = ""; - sourceAudioElementRevokersRef.current.get(id)?.(); - sourceAudioElementRevokersRef.current.delete(id); - sourceAudioElementResourcesRef.current.delete(id); - existing.delete(id); - } - } - - for (const audioPath of previewSourceAudioFallbackPaths) { - let audio = existing.get(audioPath); - if (!audio) { - audio = new Audio(); - audio.preload = "auto"; - existing.set(audioPath, audio); - } - audio.dataset.sourceAudioPath = audioPath; - - if (sourceAudioElementResourcesRef.current.get(audioPath) !== audioPath) { - audio.pause(); - audio.src = ""; - sourceAudioElementRevokersRef.current.get(audioPath)?.(); - sourceAudioElementRevokersRef.current.delete(audioPath); - sourceAudioElementResourcesRef.current.set(audioPath, audioPath); - - void (async () => { - try { - const resolved = await resolveMediaElementSource(audioPath); - const latestAudio = existing.get(audioPath); - - if ( - cancelled || - latestAudio !== audio || - sourceAudioElementResourcesRef.current.get(audioPath) !== audioPath - ) { - resolved.revoke(); - return; - } - - sourceAudioElementRevokersRef.current.set(audioPath, resolved.revoke); - latestAudio.src = resolved.src; - } catch (error) { - if (cancelled) { - return; - } - - sourceAudioElementRevokersRef.current.get(audioPath)?.(); - sourceAudioElementRevokersRef.current.delete(audioPath); - sourceAudioElementResourcesRef.current.delete(audioPath); - const latestAudio = existing.get(audioPath); - if (latestAudio === audio) { - latestAudio.pause(); - latestAudio.src = ""; - } - toast.warning( - `Could not load companion audio source: ${summarizeErrorMessage(getErrorMessage(error))}`, - { id: SOURCE_AUDIO_FALLBACK_TOAST_ID, duration: 10000 }, - ); - } - })(); - } - - audio.volume = Math.max(0, Math.min(1, previewVolume)); - } - - if (previewSourceAudioFallbackPaths.length === 0) { - lastSourceAudioSyncTimeRef.current = null; - } - - return () => { - cancelled = true; - }; - }, [previewSourceAudioFallbackPaths, previewVolume]); - - useEffect(() => { - return () => { - for (const audio of audioElementsRef.current.values()) { - audio.pause(); - audio.src = ""; - } - for (const revoke of audioElementRevokersRef.current.values()) { - revoke(); - } - audioElementsRef.current.clear(); - audioElementRevokersRef.current.clear(); - audioElementResourcesRef.current.clear(); - for (const audio of sourceAudioElementsRef.current.values()) { - audio.pause(); - audio.src = ""; - } - for (const revoke of sourceAudioElementRevokersRef.current.values()) { - revoke(); - } - sourceAudioElementsRef.current.clear(); - sourceAudioElementRevokersRef.current.clear(); - sourceAudioElementResourcesRef.current.clear(); - lastSourceAudioSyncTimeRef.current = null; - }; - }, []); - - // Sync audio playback with video currentTime and isPlaying state - useEffect(() => { - const currentTimeMs = currentTime * 1000; - const activeSpeedRegion = effectiveSpeedRegions.find( - (region) => currentTimeMs >= region.startMs && currentTimeMs < region.endMs, - ); - const targetPlaybackRate = activeSpeedRegion ? activeSpeedRegion.speed : 1; - - for (const region of audioRegions) { - const audio = audioElementsRef.current.get(region.id); - if (!audio) continue; - - const isInRegion = currentTimeMs >= region.startMs && currentTimeMs < region.endMs; - - if (isPlaying && isInRegion) { - enablePitchPreservingPlayback(audio); - const audioOffset = (currentTimeMs - region.startMs) / 1000; - // Only seek if significantly out of sync (> 200ms) - if (Math.abs(audio.currentTime - audioOffset) > 0.2) { - audio.currentTime = audioOffset; - } - const syncedPlaybackRate = getMediaSyncPlaybackRate({ - basePlaybackRate: targetPlaybackRate, - currentTime: audio.currentTime, - targetTime: audioOffset, - }); - if (Math.abs(audio.playbackRate - syncedPlaybackRate) > 0.001) { - audio.playbackRate = syncedPlaybackRate; - } - if (audio.paused) { - audio.play().catch(() => undefined); - } - } else { - if (!audio.paused) { - audio.pause(); - } - } - } - }, [isPlaying, currentTime, audioRegions, effectiveSpeedRegions]); - - useEffect(() => { - if (previewSourceAudioFallbackPaths.length === 0) { - lastSourceAudioSyncTimeRef.current = null; - return; - } - - const activeSpeedRegion = effectiveSpeedRegions.find( - (region) => currentTime * 1000 >= region.startMs && currentTime * 1000 < region.endMs, - ); - const targetPlaybackRate = activeSpeedRegion ? activeSpeedRegion.speed : 1; - const previousTimelineTime = lastSourceAudioSyncTimeRef.current; - const timelineJumped = - previousTimelineTime === null || Math.abs(currentTime - previousTimelineTime) > 0.25; - const driftThreshold = isPlaying - ? SOURCE_AUDIO_PREVIEW_PLAYING_SEEK_DRIFT_SECONDS - : SOURCE_AUDIO_PREVIEW_PAUSED_SEEK_DRIFT_SECONDS; - - for (const audio of sourceAudioElementsRef.current.values()) { - enablePitchPreservingPlayback(audio); - const audioDuration = Number.isFinite(audio.duration) ? audio.duration : null; - const startDelaySeconds = estimateCompanionAudioStartDelaySeconds( - duration, - audioDuration, - sourceAudioFallbackStartDelayMsByPath[audio.dataset.sourceAudioPath ?? ""], - ); - const beforeAudioStart = currentTime + 0.001 < startDelaySeconds; - const targetTime = clampMediaTimeToDuration( - currentTime - startDelaySeconds, - audioDuration, - ); - - if (timelineJumped || Math.abs(audio.currentTime - targetTime) > driftThreshold) { - try { - audio.currentTime = targetTime; - } catch { - // no-op - } - } - - const syncedPlaybackRate = getMediaSyncPlaybackRate({ - basePlaybackRate: targetPlaybackRate, - currentTime: audio.currentTime, - targetTime, - toleranceSeconds: SOURCE_AUDIO_PREVIEW_RATE_TOLERANCE_SECONDS, - correctionWindowSeconds: SOURCE_AUDIO_PREVIEW_RATE_CORRECTION_WINDOW_SECONDS, - maxAdjustment: SOURCE_AUDIO_PREVIEW_MAX_RATE_ADJUSTMENT, - }); - if (Math.abs(audio.playbackRate - syncedPlaybackRate) > 0.001) { - audio.playbackRate = syncedPlaybackRate; - } - - const atEnd = audioDuration !== null && targetTime >= audioDuration; - if (isPlaying && !beforeAudioStart && !atEnd) { - audio.play().catch(() => undefined); - } else if (!audio.paused) { - audio.pause(); - } - } - - lastSourceAudioSyncTimeRef.current = currentTime; - }, [ + useAudioPreviewSync({ + audioRegions, + previewVolume, + isPlaying, currentTime, duration, - isPlaying, + effectiveSpeedRegions, previewSourceAudioFallbackPaths, sourceAudioFallbackStartDelayMsByPath, - effectiveSpeedRegions, - ]); + isCurrentClipMuted, + getSourceTrackPreviewGain, + onSourceFallbackLoadError: (error) => { + toast.warning( + `Could not load companion audio source: ${summarizeErrorMessage(getErrorMessage(error))}`, + { duration: 10000 }, + ); + }, + }); const showExportSuccessToast = useCallback((filePath: string) => { toast.success(`Exported successfully to ${filePath}`, { @@ -6049,6 +5762,17 @@ export default function VideoEditor() { selectedClipId && handleClipMutedChange(muted) } onClipDelete={handleClipDelete} + hasClipSourceAudio={hasClipSourceAudio} + showClipSourceAudioTrack={showClipSourceAudioTrack} + onShowClipSourceAudioTrackChange={setShowClipSourceAudioTrack} + sourceAudioTrackMeta={sourceAudioTrackMeta} + sourceAudioTrackSettings={selectedClipSourceAudioTrackSettings} + onSourceAudioTrackVolumeChange={ + onSelectedClipSourceAudioTrackVolumeChange + } + onSourceAudioTrackNormalizeChange={ + onSelectedClipSourceAudioTrackNormalizeChange + } selectedAudioId={selectedAudioId} selectedAudioVolume={ selectedAudioId @@ -6353,7 +6077,17 @@ export default function VideoEditor() { cursorClickBounceDuration } cursorSway={cursorSway} - volume={shouldMutePreviewVideo ? 0 : previewVolume} + volume={ + shouldMutePreviewVideo || isCurrentClipMuted + ? 0 + : Math.max( + 0, + Math.min( + 1, + previewVolume * embeddedSourcePreviewGain, + ), + ) + } suspendRendering={shouldSuspendPreviewRendering} /> @@ -6628,6 +6362,17 @@ export default function VideoEditor() { selectedAnnotationId={selectedAnnotationId} onSelectAnnotation={handleSelectAnnotation} aspectRatio={aspectRatio} + showSourceAudioTrack={showClipSourceAudioTrack} + sourceAudioTrackSettings={activeSourceAudioTrackSettings} + onSourceAudioAvailabilityChange={(available) => { + setHasClipSourceAudio(available); + if (!available) { + setShowClipSourceAudioTrack(false); + } + }} + onSourceAudioTracksMetaChange={(tracks) => { + onSourceAudioTracksMetaChange(tracks); + }} /> diff --git a/src/components/video-editor/audio/clipAudio.ts b/src/components/video-editor/audio/clipAudio.ts new file mode 100644 index 00000000..d539ad4b --- /dev/null +++ b/src/components/video-editor/audio/clipAudio.ts @@ -0,0 +1,19 @@ +import { mapSourceTimeToTimelineTime } from "../types"; +import type { ClipRegion } from "../types"; + +export function getActiveClipIdAtSourceTime( + sourceTimeSeconds: number, + clipRegions: ClipRegion[], +): string | null { + const sourceMs = sourceTimeSeconds * 1000; + const timelineMs = mapSourceTimeToTimelineTime(sourceMs, clipRegions); + const activeClip = clipRegions.find( + (clip) => timelineMs >= clip.startMs && timelineMs < clip.endMs, + ); + return activeClip?.id ?? null; +} + +export function isClipMutedById(clipId: string | null, clipRegions: ClipRegion[]): boolean { + if (!clipId) return false; + return clipRegions.find((clip) => clip.id === clipId)?.muted ?? false; +} diff --git a/src/components/video-editor/audio/sourceAudioTracks.ts b/src/components/video-editor/audio/sourceAudioTracks.ts new file mode 100644 index 00000000..8b9998c3 --- /dev/null +++ b/src/components/video-editor/audio/sourceAudioTracks.ts @@ -0,0 +1,9 @@ +export const SOURCE_AUDIO_FALLBACK_TOAST_ID = "source-audio-fallback-error"; +export const SOURCE_AUDIO_NORMALIZE_GAIN = 1.35; + +export function getSourceTrackIdFromPath(audioPath: string): "mic" | "system" | "mixed" { + const normalized = audioPath.toLowerCase(); + if (normalized.includes(".mic.")) return "mic"; + if (normalized.includes(".system.")) return "system"; + return "mixed"; +} diff --git a/src/components/video-editor/audio/useAudioPreviewSync.ts b/src/components/video-editor/audio/useAudioPreviewSync.ts new file mode 100644 index 00000000..620ae2e6 --- /dev/null +++ b/src/components/video-editor/audio/useAudioPreviewSync.ts @@ -0,0 +1,322 @@ +import { useEffect, useRef } from "react"; +import { resolveMediaElementSource } from "@/lib/exporter/localMediaSource"; +import { + clampMediaTimeToDuration, + enablePitchPreservingPlayback, + estimateCompanionAudioStartDelaySeconds, + getMediaSyncPlaybackRate, +} from "@/lib/mediaTiming"; +import type { AudioRegion, SpeedRegion } from "../types"; + +const SOURCE_AUDIO_PREVIEW_PLAYING_SEEK_DRIFT_SECONDS = 0.18; +const SOURCE_AUDIO_PREVIEW_PAUSED_SEEK_DRIFT_SECONDS = 0.01; +const SOURCE_AUDIO_PREVIEW_RATE_TOLERANCE_SECONDS = 0.08; +const SOURCE_AUDIO_PREVIEW_RATE_CORRECTION_WINDOW_SECONDS = 8; +const SOURCE_AUDIO_PREVIEW_MAX_RATE_ADJUSTMENT = 0.015; + +interface UseAudioPreviewSyncParams { + audioRegions: AudioRegion[]; + previewVolume: number; + isPlaying: boolean; + currentTime: number; + duration: number; + effectiveSpeedRegions: SpeedRegion[]; + previewSourceAudioFallbackPaths: string[]; + sourceAudioFallbackStartDelayMsByPath: Record; + isCurrentClipMuted: boolean; + getSourceTrackPreviewGain: (audioPath: string) => number; + onSourceFallbackLoadError: (error: unknown) => void; +} + +export function useAudioPreviewSync({ + audioRegions, + previewVolume, + isPlaying, + currentTime, + duration, + effectiveSpeedRegions, + previewSourceAudioFallbackPaths, + sourceAudioFallbackStartDelayMsByPath, + isCurrentClipMuted, + getSourceTrackPreviewGain, + onSourceFallbackLoadError, +}: UseAudioPreviewSyncParams) { + const audioElementsRef = useRef>(new Map()); + const audioElementRevokersRef = useRef void>>(new Map()); + const audioElementResourcesRef = useRef>(new Map()); + const sourceAudioElementsRef = useRef>(new Map()); + const sourceAudioElementRevokersRef = useRef void>>(new Map()); + const sourceAudioElementResourcesRef = useRef>(new Map()); + const lastSourceAudioSyncTimeRef = useRef(null); + + useEffect(() => { + let cancelled = false; + const existing = audioElementsRef.current; + const currentIds = new Set(audioRegions.map((r) => r.id)); + + for (const [id, audio] of existing) { + if (!currentIds.has(id)) { + audio.pause(); + audio.src = ""; + audioElementRevokersRef.current.get(id)?.(); + audioElementRevokersRef.current.delete(id); + audioElementResourcesRef.current.delete(id); + existing.delete(id); + } + } + + for (const region of audioRegions) { + let audio = existing.get(region.id); + if (!audio) { + audio = new Audio(); + audio.preload = "auto"; + existing.set(region.id, audio); + } + + if (audioElementResourcesRef.current.get(region.id) !== region.audioPath) { + audio.pause(); + audio.src = ""; + audioElementRevokersRef.current.get(region.id)?.(); + audioElementRevokersRef.current.delete(region.id); + audioElementResourcesRef.current.set(region.id, region.audioPath); + + void (async () => { + const resolved = await resolveMediaElementSource(region.audioPath); + const latestAudio = existing.get(region.id); + + if ( + cancelled || + latestAudio !== audio || + audioElementResourcesRef.current.get(region.id) !== region.audioPath + ) { + resolved.revoke(); + return; + } + + audioElementRevokersRef.current.set(region.id, resolved.revoke); + latestAudio.src = resolved.src; + })(); + } + + audio.volume = Math.max(0, Math.min(1, region.volume * previewVolume)); + } + + return () => { + cancelled = true; + }; + }, [audioRegions, previewVolume]); + + useEffect(() => { + let cancelled = false; + const existing = sourceAudioElementsRef.current; + const currentIds = new Set(previewSourceAudioFallbackPaths); + + for (const [id, audio] of existing) { + if (!currentIds.has(id)) { + audio.pause(); + audio.src = ""; + sourceAudioElementRevokersRef.current.get(id)?.(); + sourceAudioElementRevokersRef.current.delete(id); + sourceAudioElementResourcesRef.current.delete(id); + existing.delete(id); + } + } + + for (const audioPath of previewSourceAudioFallbackPaths) { + let audio = existing.get(audioPath); + if (!audio) { + audio = new Audio(); + audio.preload = "auto"; + existing.set(audioPath, audio); + } + audio.dataset.sourceAudioPath = audioPath; + + if (sourceAudioElementResourcesRef.current.get(audioPath) !== audioPath) { + audio.pause(); + audio.src = ""; + sourceAudioElementRevokersRef.current.get(audioPath)?.(); + sourceAudioElementRevokersRef.current.delete(audioPath); + sourceAudioElementResourcesRef.current.set(audioPath, audioPath); + + void (async () => { + try { + const resolved = await resolveMediaElementSource(audioPath); + const latestAudio = existing.get(audioPath); + + if ( + cancelled || + latestAudio !== audio || + sourceAudioElementResourcesRef.current.get(audioPath) !== audioPath + ) { + resolved.revoke(); + return; + } + + sourceAudioElementRevokersRef.current.set(audioPath, resolved.revoke); + latestAudio.src = resolved.src; + } catch (error) { + if (cancelled) { + return; + } + + sourceAudioElementRevokersRef.current.get(audioPath)?.(); + sourceAudioElementRevokersRef.current.delete(audioPath); + sourceAudioElementResourcesRef.current.delete(audioPath); + const latestAudio = existing.get(audioPath); + if (latestAudio === audio) { + latestAudio.pause(); + latestAudio.src = ""; + } + onSourceFallbackLoadError(error); + } + })(); + } + + audio.volume = isCurrentClipMuted + ? 0 + : Math.max(0, Math.min(1, previewVolume * getSourceTrackPreviewGain(audioPath))); + } + + if (previewSourceAudioFallbackPaths.length === 0) { + lastSourceAudioSyncTimeRef.current = null; + } + + return () => { + cancelled = true; + }; + }, [ + getSourceTrackPreviewGain, + isCurrentClipMuted, + onSourceFallbackLoadError, + previewSourceAudioFallbackPaths, + previewVolume, + ]); + + useEffect(() => { + return () => { + for (const audio of audioElementsRef.current.values()) { + audio.pause(); + audio.src = ""; + } + for (const revoke of audioElementRevokersRef.current.values()) { + revoke(); + } + audioElementsRef.current.clear(); + audioElementRevokersRef.current.clear(); + audioElementResourcesRef.current.clear(); + for (const audio of sourceAudioElementsRef.current.values()) { + audio.pause(); + audio.src = ""; + } + for (const revoke of sourceAudioElementRevokersRef.current.values()) { + revoke(); + } + sourceAudioElementsRef.current.clear(); + sourceAudioElementRevokersRef.current.clear(); + sourceAudioElementResourcesRef.current.clear(); + lastSourceAudioSyncTimeRef.current = null; + }; + }, []); + + useEffect(() => { + const currentTimeMs = currentTime * 1000; + const activeSpeedRegion = effectiveSpeedRegions.find( + (region) => currentTimeMs >= region.startMs && currentTimeMs < region.endMs, + ); + const targetPlaybackRate = activeSpeedRegion ? activeSpeedRegion.speed : 1; + + for (const region of audioRegions) { + const audio = audioElementsRef.current.get(region.id); + if (!audio) continue; + + const isInRegion = currentTimeMs >= region.startMs && currentTimeMs < region.endMs; + + if (isPlaying && isInRegion) { + enablePitchPreservingPlayback(audio); + const audioOffset = (currentTimeMs - region.startMs) / 1000; + if (Math.abs(audio.currentTime - audioOffset) > 0.2) { + audio.currentTime = audioOffset; + } + const syncedPlaybackRate = getMediaSyncPlaybackRate({ + basePlaybackRate: targetPlaybackRate, + currentTime: audio.currentTime, + targetTime: audioOffset, + }); + if (Math.abs(audio.playbackRate - syncedPlaybackRate) > 0.001) { + audio.playbackRate = syncedPlaybackRate; + } + if (audio.paused) { + audio.play().catch(() => undefined); + } + } else if (!audio.paused) { + audio.pause(); + } + } + }, [audioRegions, currentTime, effectiveSpeedRegions, isPlaying]); + + useEffect(() => { + if (previewSourceAudioFallbackPaths.length === 0) { + lastSourceAudioSyncTimeRef.current = null; + return; + } + + const activeSpeedRegion = effectiveSpeedRegions.find( + (region) => currentTime * 1000 >= region.startMs && currentTime * 1000 < region.endMs, + ); + const targetPlaybackRate = activeSpeedRegion ? activeSpeedRegion.speed : 1; + const previousTimelineTime = lastSourceAudioSyncTimeRef.current; + const timelineJumped = + previousTimelineTime === null || Math.abs(currentTime - previousTimelineTime) > 0.25; + const driftThreshold = isPlaying + ? SOURCE_AUDIO_PREVIEW_PLAYING_SEEK_DRIFT_SECONDS + : SOURCE_AUDIO_PREVIEW_PAUSED_SEEK_DRIFT_SECONDS; + + for (const audio of sourceAudioElementsRef.current.values()) { + enablePitchPreservingPlayback(audio); + const audioDuration = Number.isFinite(audio.duration) ? audio.duration : null; + const startDelaySeconds = estimateCompanionAudioStartDelaySeconds( + duration, + audioDuration, + sourceAudioFallbackStartDelayMsByPath[audio.dataset.sourceAudioPath ?? ""], + ); + const beforeAudioStart = currentTime + 0.001 < startDelaySeconds; + const targetTime = clampMediaTimeToDuration(currentTime - startDelaySeconds, audioDuration); + + if (timelineJumped || Math.abs(audio.currentTime - targetTime) > driftThreshold) { + try { + audio.currentTime = targetTime; + } catch { + // no-op + } + } + + const syncedPlaybackRate = getMediaSyncPlaybackRate({ + basePlaybackRate: targetPlaybackRate, + currentTime: audio.currentTime, + targetTime, + toleranceSeconds: SOURCE_AUDIO_PREVIEW_RATE_TOLERANCE_SECONDS, + correctionWindowSeconds: SOURCE_AUDIO_PREVIEW_RATE_CORRECTION_WINDOW_SECONDS, + maxAdjustment: SOURCE_AUDIO_PREVIEW_MAX_RATE_ADJUSTMENT, + }); + if (Math.abs(audio.playbackRate - syncedPlaybackRate) > 0.001) { + audio.playbackRate = syncedPlaybackRate; + } + + const atEnd = audioDuration !== null && targetTime >= audioDuration; + if (isPlaying && !beforeAudioStart && !atEnd) { + audio.play().catch(() => undefined); + } else if (!audio.paused) { + audio.pause(); + } + } + + lastSourceAudioSyncTimeRef.current = currentTime; + }, [ + currentTime, + duration, + effectiveSpeedRegions, + isPlaying, + previewSourceAudioFallbackPaths, + sourceAudioFallbackStartDelayMsByPath, + ]); +} diff --git a/src/components/video-editor/audio/useSourceAudioFallback.ts b/src/components/video-editor/audio/useSourceAudioFallback.ts new file mode 100644 index 00000000..64b4caa8 --- /dev/null +++ b/src/components/video-editor/audio/useSourceAudioFallback.ts @@ -0,0 +1,68 @@ +import { useEffect, useState } from "react"; +import { toast } from "sonner"; +import { SOURCE_AUDIO_FALLBACK_TOAST_ID } from "./sourceAudioTracks"; + +interface UseSourceAudioFallbackParams { + currentSourcePath: string | null; + summarizeErrorMessage: (message: string) => string; +} + +export function useSourceAudioFallback({ + currentSourcePath, + summarizeErrorMessage, +}: UseSourceAudioFallbackParams) { + const [sourceAudioFallbackPaths, setSourceAudioFallbackPaths] = useState([]); + const [sourceAudioFallbackStartDelayMsByPath, setSourceAudioFallbackStartDelayMsByPath] = + useState>({}); + + useEffect(() => { + let cancelled = false; + setSourceAudioFallbackPaths([]); + setSourceAudioFallbackStartDelayMsByPath({}); + + if (!currentSourcePath) { + return () => { + cancelled = true; + }; + } + + void (async () => { + try { + const result = await window.electronAPI.getVideoAudioFallbackPaths(currentSourcePath); + if (cancelled) { + return; + } + if (!result.success) { + setSourceAudioFallbackPaths([]); + setSourceAudioFallbackStartDelayMsByPath({}); + toast.warning( + result.error + ? `Could not load companion audio sources: ${summarizeErrorMessage(result.error)}` + : "Could not load companion audio sources. Playback and export may miss microphone audio.", + { id: SOURCE_AUDIO_FALLBACK_TOAST_ID, duration: 10000 }, + ); + return; + } + + toast.dismiss(SOURCE_AUDIO_FALLBACK_TOAST_ID); + setSourceAudioFallbackPaths(result.paths ?? []); + setSourceAudioFallbackStartDelayMsByPath(result.startDelayMsByPath ?? {}); + } catch (error) { + if (!cancelled) { + setSourceAudioFallbackPaths([]); + setSourceAudioFallbackStartDelayMsByPath({}); + toast.warning( + `Could not load companion audio sources: ${summarizeErrorMessage(String(error))}`, + { id: SOURCE_AUDIO_FALLBACK_TOAST_ID, duration: 10000 }, + ); + } + } + })(); + + return () => { + cancelled = true; + }; + }, [currentSourcePath, summarizeErrorMessage]); + + return { sourceAudioFallbackPaths, sourceAudioFallbackStartDelayMsByPath }; +} diff --git a/src/components/video-editor/audio/useSourceAudioTrackSettings.ts b/src/components/video-editor/audio/useSourceAudioTrackSettings.ts new file mode 100644 index 00000000..27ea0daf --- /dev/null +++ b/src/components/video-editor/audio/useSourceAudioTrackSettings.ts @@ -0,0 +1,112 @@ +import { useCallback, useMemo, useState } from "react"; + +export type SourceAudioTrackSetting = { volume: number; normalize: boolean }; +export type SourceAudioTrackSettings = Record; +export type SourceAudioTrackMeta = Array<{ id: string; label: string }>; + +interface UseSourceAudioTrackSettingsParams { + selectedClipId: string | null; + activeClipId: string | null; +} + +export interface UseSourceAudioTrackSettingsResult { + sourceAudioTrackMeta: SourceAudioTrackMeta; + activeSourceAudioTrackSettings: SourceAudioTrackSettings; + selectedClipSourceAudioTrackSettings: SourceAudioTrackSettings; + onSourceAudioTracksMetaChange: (tracks: SourceAudioTrackMeta) => void; + onSelectedClipSourceAudioTrackVolumeChange: (id: string, volume: number) => void; + onSelectedClipSourceAudioTrackNormalizeChange: (id: string, normalize: boolean) => void; +} + +export function useSourceAudioTrackSettings({ + selectedClipId, + activeClipId, +}: UseSourceAudioTrackSettingsParams): UseSourceAudioTrackSettingsResult { + const [sourceAudioTrackMeta, setSourceAudioTrackMeta] = useState([]); + const [sourceAudioTrackSettingsByClip, setSourceAudioTrackSettingsByClip] = useState< + Record + >({}); + const [defaultSourceAudioTrackSettings, setDefaultSourceAudioTrackSettings] = useState< + SourceAudioTrackSettings + >({}); + + const activeSourceAudioTrackSettings = useMemo(() => { + if (!activeClipId) { + return defaultSourceAudioTrackSettings; + } + return { + ...defaultSourceAudioTrackSettings, + ...(sourceAudioTrackSettingsByClip[activeClipId] ?? {}), + }; + }, [activeClipId, defaultSourceAudioTrackSettings, sourceAudioTrackSettingsByClip]); + + const selectedClipSourceAudioTrackSettings = useMemo(() => { + if (!selectedClipId) { + return defaultSourceAudioTrackSettings; + } + return { + ...defaultSourceAudioTrackSettings, + ...(sourceAudioTrackSettingsByClip[selectedClipId] ?? {}), + }; + }, [defaultSourceAudioTrackSettings, selectedClipId, sourceAudioTrackSettingsByClip]); + + const onSourceAudioTracksMetaChange = useCallback((tracks: SourceAudioTrackMeta) => { + setSourceAudioTrackMeta(tracks); + setDefaultSourceAudioTrackSettings((prev) => { + const next: SourceAudioTrackSettings = {}; + for (const track of tracks) { + next[track.id] = prev[track.id] ?? { volume: 1, normalize: false }; + } + return next; + }); + }, []); + + const onSelectedClipSourceAudioTrackVolumeChange = useCallback( + (id: string, volume: number) => { + if (!selectedClipId) return; + setSourceAudioTrackSettingsByClip((prev) => { + const prevClip = prev[selectedClipId] ?? defaultSourceAudioTrackSettings; + return { + ...prev, + [selectedClipId]: { + ...prevClip, + [id]: { + volume: Math.max(0, Math.min(2, volume)), + normalize: prevClip[id]?.normalize ?? false, + }, + }, + }; + }); + }, + [defaultSourceAudioTrackSettings, selectedClipId], + ); + + const onSelectedClipSourceAudioTrackNormalizeChange = useCallback( + (id: string, normalize: boolean) => { + if (!selectedClipId) return; + setSourceAudioTrackSettingsByClip((prev) => { + const prevClip = prev[selectedClipId] ?? defaultSourceAudioTrackSettings; + return { + ...prev, + [selectedClipId]: { + ...prevClip, + [id]: { + volume: prevClip[id]?.volume ?? 1, + normalize, + }, + }, + }; + }); + }, + [defaultSourceAudioTrackSettings, selectedClipId], + ); + + return { + sourceAudioTrackMeta, + activeSourceAudioTrackSettings, + selectedClipSourceAudioTrackSettings, + onSourceAudioTracksMetaChange, + onSelectedClipSourceAudioTrackVolumeChange, + onSelectedClipSourceAudioTrackNormalizeChange, + }; +} diff --git a/src/components/video-editor/timeline/components/waveform/WaveformGenerator.ts b/src/components/video-editor/audio/waveform/WaveformGenerator.ts similarity index 95% rename from src/components/video-editor/timeline/components/waveform/WaveformGenerator.ts rename to src/components/video-editor/audio/waveform/WaveformGenerator.ts index 128a65ad..a51475d8 100644 --- a/src/components/video-editor/timeline/components/waveform/WaveformGenerator.ts +++ b/src/components/video-editor/audio/waveform/WaveformGenerator.ts @@ -1,6 +1,6 @@ import WorkerConstructor from "./waveform.worker?worker"; -import type { AudioPeaksData } from "../../core/timelineTypes"; -import { WAVEFORM_DEFAULT_PEAK_COUNT } from "../../core/constants"; +import type { AudioPeaksData } from "../../timeline/core/timelineTypes"; +import { WAVEFORM_DEFAULT_PEAK_COUNT } from "../../timeline/core/constants"; export class WaveformGenerator { private audioContext: AudioContext; diff --git a/src/components/video-editor/timeline/components/waveform/waveform.worker.ts b/src/components/video-editor/audio/waveform/waveform.worker.ts similarity index 100% rename from src/components/video-editor/timeline/components/waveform/waveform.worker.ts rename to src/components/video-editor/audio/waveform/waveform.worker.ts diff --git a/src/components/video-editor/timeline/TimelineEditor.tsx b/src/components/video-editor/timeline/TimelineEditor.tsx index cf7005e4..6fc22499 100644 --- a/src/components/video-editor/timeline/TimelineEditor.tsx +++ b/src/components/video-editor/timeline/TimelineEditor.tsx @@ -17,6 +17,7 @@ import { } from "@/utils/aspectRatioUtils"; import { formatShortcut } from "@/utils/platformUtils"; import { loadEditorPreferences, saveEditorPreferences } from "../editorPreferences"; +import { fromFileUrl } from "../projectPersistence"; import type { AnnotationRegion, AudioRegion, @@ -35,6 +36,7 @@ import { useTimelineEditorRuntime } from "./hooks/useTimelineEditorRuntime"; import { useTimelineRange } from "./hooks/useTimelineRange"; import TimelineCanvas from "./components/viewport/TimelineCanvas"; import TimelineToolbar from "./components/toolbar/TimelineToolbar"; +import type { AudioPeaksData } from "./core/timelineTypes"; export interface TimelineEditorProps { videoDuration: number; @@ -80,6 +82,35 @@ export interface TimelineEditorProps { isCropped?: boolean; videoPath?: string | null; hideToolbar?: boolean; + showSourceAudioTrack?: boolean; + onSourceAudioAvailabilityChange?: (available: boolean) => void; + sourceAudioTrackSettings?: Record; + onSourceAudioTracksMetaChange?: (tracks: Array<{ id: string; label: string }>) => void; +} + +function extractLocalPathFromMediaServerUrl(input: string | null | undefined): string | null { + if (!input) return null; + try { + const url = new URL(input); + const isLocalMediaServer = + (url.protocol === "http:" || url.protocol === "https:") && + (url.hostname === "127.0.0.1" || url.hostname === "localhost") && + url.pathname === "/video"; + if (!isLocalMediaServer) return null; + return url.searchParams.get("path"); + } catch { + return null; + } +} + +function buildSourceSidecarPath(source: string, suffix: "mic" | "system"): string { + const normalized = source.replace(/\\/g, "/"); + const lastSlash = normalized.lastIndexOf("/"); + const dir = lastSlash >= 0 ? normalized.slice(0, lastSlash + 1) : ""; + const fileName = lastSlash >= 0 ? normalized.slice(lastSlash + 1) : normalized; + const dotIndex = fileName.lastIndexOf("."); + const baseName = dotIndex > 0 ? fileName.slice(0, dotIndex) : fileName; + return `${dir}${baseName}.${suffix}.wav`; } export interface TimelineEditorHandle { @@ -138,6 +169,10 @@ const TimelineEditor = forwardRef( isCropped = false, videoPath, hideToolbar = false, + showSourceAudioTrack = false, + onSourceAudioAvailabilityChange, + sourceAudioTrackSettings = {}, + onSourceAudioTracksMetaChange, }, ref, ) { @@ -229,9 +264,65 @@ const TimelineEditor = forwardRef( return { previewSpans, hiddenZoomIds }; }, [clipRegions, liveSpanPreviewById, zoomRegions]); const { shortcuts: keyShortcuts, isMac } = useShortcuts(); - const audioPeaks = useTimelineAudioPeaks(videoPath, { + const sourceAudioPeaks = useTimelineAudioPeaks(videoPath, { enableSourceSidecarFallback: true, }); + const localSourcePath = useMemo(() => { + if (!videoPath) return null; + return ( + extractLocalPathFromMediaServerUrl(videoPath) || + (/^file:\/\//i.test(videoPath) ? fromFileUrl(videoPath) : videoPath) + ); + }, [videoPath]); + const micSidecarPath = useMemo( + () => (localSourcePath ? buildSourceSidecarPath(localSourcePath, "mic") : null), + [localSourcePath], + ); + const systemSidecarPath = useMemo( + () => (localSourcePath ? buildSourceSidecarPath(localSourcePath, "system") : null), + [localSourcePath], + ); + const micSidecarPeaks = useTimelineAudioPeaks(micSidecarPath); + const systemSidecarPeaks = useTimelineAudioPeaks(systemSidecarPath); + const sourceAudioTracks = useMemo>(() => { + if (systemSidecarPeaks || micSidecarPeaks) { + const tracks: Array<{ id: string; label: string; peaks: AudioPeaksData }> = []; + if (systemSidecarPeaks) tracks.push({ id: "system", label: "Source System", peaks: systemSidecarPeaks }); + if (micSidecarPeaks) tracks.push({ id: "mic", label: "Source Mic", peaks: micSidecarPeaks }); + return tracks; + } + return sourceAudioPeaks ? [{ id: "mixed", label: "Source", peaks: sourceAudioPeaks }] : []; + }, [micSidecarPeaks, sourceAudioPeaks, systemSidecarPeaks]); + useEffect(() => { + onSourceAudioTracksMetaChange?.(sourceAudioTracks.map((t) => ({ id: t.id, label: t.label }))); + }, [onSourceAudioTracksMetaChange, sourceAudioTracks]); + const displaySourceAudioTracks = useMemo(() => { + return sourceAudioTracks.map((track) => { + const settings = sourceAudioTrackSettings[track.id] ?? { volume: 1, normalize: false }; + const volume = Math.max(0, Math.min(2, settings.volume)); + const normalize = settings.normalize; + const input = track.peaks.peaks; + const adjusted = new Float32Array(input.length); + for (let i = 0; i < input.length; i++) { + let amp = input[i]; + if (normalize) { + amp = Math.sqrt(amp); + } + adjusted[i] = Math.max(0, Math.min(1, amp * volume)); + } + return { + id: track.id, + label: track.label, + peaks: { + durationMs: track.peaks.durationMs, + peaks: adjusted, + } satisfies AudioPeaksData, + }; + }); + }, [sourceAudioTrackSettings, sourceAudioTracks]); + useEffect(() => { + onSourceAudioAvailabilityChange?.(sourceAudioTracks.length > 0); + }, [onSourceAudioAvailabilityChange, sourceAudioTracks.length]); useEffect(() => { if (aspectRatio === "native") { @@ -474,7 +565,8 @@ const TimelineEditor = forwardRef( selectAllBlocksActive={selectAllBlocksActive} onClearBlockSelection={clearSelectedBlocks} keyframes={keyframes} - audioPeaks={audioPeaks} + sourceAudioTracks={displaySourceAudioTracks} + showSourceAudioTrack={showSourceAudioTrack} liveSpanPreviewById={liveZoomPreview.previewSpans} liveHiddenItemIds={Array.from(liveZoomPreview.hiddenZoomIds)} /> diff --git a/src/components/video-editor/timeline/components/viewport/TimelineCanvas.tsx b/src/components/video-editor/timeline/components/viewport/TimelineCanvas.tsx index d0dae8c2..94d4cd88 100644 --- a/src/components/video-editor/timeline/components/viewport/TimelineCanvas.tsx +++ b/src/components/video-editor/timeline/components/viewport/TimelineCanvas.tsx @@ -57,7 +57,8 @@ interface TimelineCanvasProps { selectAllBlocksActive?: boolean; onClearBlockSelection?: () => void; keyframes?: { id: string; time: number }[]; - audioPeaks?: AudioPeaksData | null; + sourceAudioTracks?: Array<{ id: string; label: string; peaks: AudioPeaksData }>; + showSourceAudioTrack?: boolean; liveSpanPreviewById?: Record; liveHiddenItemIds?: string[]; } @@ -220,7 +221,8 @@ interface TimelineCanvasRowsProps { onSelectClip?: (id: string | null) => void; onSelectAnnotation?: (id: string | null) => void; onSelectAudio?: (id: string | null) => void; - audioPeaks?: AudioPeaksData | null; + sourceAudioTracks?: Array<{ id: string; label: string; peaks: AudioPeaksData }>; + showSourceAudioTrack?: boolean; liveSpanPreviewById?: Record; liveHiddenItemIds?: string[]; direction: string; @@ -282,7 +284,8 @@ const TimelineCanvasRows = memo(function TimelineCanvasRows({ onSelectClip, onSelectAnnotation, onSelectAudio, - audioPeaks, + sourceAudioTracks = [], + showSourceAudioTrack = false, liveSpanPreviewById, liveHiddenItemIds, direction, @@ -365,26 +368,27 @@ const TimelineCanvasRows = memo(function TimelineCanvasRows({ ))} - {audioPeaks && ( - - {clipItems.map((item) => ( - onSelectClip?.(item.id)} - variant="audio" - waveformPeaks={audioPeaks} - waveformSegmentSpan={liveSpanPreviewById?.[item.id] ?? item.span} - > - Source - - ))} - - )} + {showSourceAudioTrack && + sourceAudioTracks.map((track) => ( + + {clipItems.map((item) => ( + onSelectClip?.(item.id)} + variant="audio" + waveformPeaks={track.peaks} + waveformSegmentSpan={liveSpanPreviewById?.[item.id] ?? item.span} + > + {track.label} + + ))} + + ))} Date: Sat, 9 May 2026 14:31:06 +0200 Subject: [PATCH 06/25] audio part seperated from video editor --- src/components/video-editor/VideoEditor.tsx | 21 ++----- .../audio/useClipAudioSettingsController.ts | 59 +++++++++++++++++++ 2 files changed, 63 insertions(+), 17 deletions(-) create mode 100644 src/components/video-editor/audio/useClipAudioSettingsController.ts diff --git a/src/components/video-editor/VideoEditor.tsx b/src/components/video-editor/VideoEditor.tsx index 6208ca3f..c7394dda 100644 --- a/src/components/video-editor/VideoEditor.tsx +++ b/src/components/video-editor/VideoEditor.tsx @@ -140,11 +140,10 @@ import { validateProjectData, } from "./projectPersistence"; import { SettingsPanel } from "./SettingsPanel"; -import { SOURCE_AUDIO_NORMALIZE_GAIN, getSourceTrackIdFromPath } from "./audio/sourceAudioTracks"; import { useAudioPreviewSync } from "./audio/useAudioPreviewSync"; +import { useClipAudioSettingsController } from "./audio/useClipAudioSettingsController"; import { useSourceAudioFallback } from "./audio/useSourceAudioFallback"; import { getActiveClipIdAtSourceTime, isClipMutedById } from "./audio/clipAudio"; -import { useSourceAudioTrackSettings } from "./audio/useSourceAudioTrackSettings"; import { APP_HEADER_ICON_BUTTON_CLASS, DiscordLinkButton, @@ -1791,24 +1790,12 @@ export default function VideoEditor() { onSourceAudioTracksMetaChange, onSelectedClipSourceAudioTrackVolumeChange, onSelectedClipSourceAudioTrackNormalizeChange, - } = useSourceAudioTrackSettings({ + embeddedSourcePreviewGain, + getSourceTrackPreviewGain, + } = useClipAudioSettingsController({ selectedClipId, activeClipId: activeClipIdAtCurrentTime, }); - const embeddedSourcePreviewGain = useMemo(() => { - const settings = activeSourceAudioTrackSettings.mixed ?? { volume: 1, normalize: false }; - const normalizeGain = settings.normalize ? SOURCE_AUDIO_NORMALIZE_GAIN : 1; - return Math.max(0, Math.min(2, settings.volume * normalizeGain)); - }, [activeSourceAudioTrackSettings]); - const getSourceTrackPreviewGain = useCallback( - (audioPath: string) => { - const trackId = getSourceTrackIdFromPath(audioPath); - const settings = activeSourceAudioTrackSettings[trackId] ?? { volume: 1, normalize: false }; - const normalizeGain = settings.normalize ? SOURCE_AUDIO_NORMALIZE_GAIN : 1; - return Math.max(0, Math.min(2, settings.volume * normalizeGain)); - }, - [activeSourceAudioTrackSettings], - ); const isCurrentClipMuted = useMemo(() => { return isClipMutedById(activeClipIdAtCurrentTime, clipRegions); }, [activeClipIdAtCurrentTime, clipRegions]); diff --git a/src/components/video-editor/audio/useClipAudioSettingsController.ts b/src/components/video-editor/audio/useClipAudioSettingsController.ts new file mode 100644 index 00000000..f6d69881 --- /dev/null +++ b/src/components/video-editor/audio/useClipAudioSettingsController.ts @@ -0,0 +1,59 @@ +import { useCallback, useMemo } from "react"; +import { + SOURCE_AUDIO_NORMALIZE_GAIN, + getSourceTrackIdFromPath, +} from "./sourceAudioTracks"; +import { useSourceAudioTrackSettings } from "./useSourceAudioTrackSettings"; + +interface UseClipAudioSettingsControllerParams { + selectedClipId: string | null; + activeClipId: string | null; +} + +export function useClipAudioSettingsController({ + selectedClipId, + activeClipId, +}: UseClipAudioSettingsControllerParams) { + const { + sourceAudioTrackMeta, + activeSourceAudioTrackSettings, + selectedClipSourceAudioTrackSettings, + onSourceAudioTracksMetaChange, + onSelectedClipSourceAudioTrackVolumeChange, + onSelectedClipSourceAudioTrackNormalizeChange, + } = useSourceAudioTrackSettings({ + selectedClipId, + activeClipId, + }); + + const embeddedSourcePreviewGain = useMemo(() => { + const settings = activeSourceAudioTrackSettings.mixed ?? { volume: 1, normalize: false }; + const normalizeGain = settings.normalize ? SOURCE_AUDIO_NORMALIZE_GAIN : 1; + return Math.max(0, Math.min(2, settings.volume * normalizeGain)); + }, [activeSourceAudioTrackSettings]); + + const getSourceTrackPreviewGain = useCallback( + (audioPath: string) => { + const trackId = getSourceTrackIdFromPath(audioPath); + const settings = activeSourceAudioTrackSettings[trackId] ?? { + volume: 1, + normalize: false, + }; + const normalizeGain = settings.normalize ? SOURCE_AUDIO_NORMALIZE_GAIN : 1; + return Math.max(0, Math.min(2, settings.volume * normalizeGain)); + }, + [activeSourceAudioTrackSettings], + ); + + return { + sourceAudioTrackMeta, + activeSourceAudioTrackSettings, + selectedClipSourceAudioTrackSettings, + onSourceAudioTracksMetaChange, + onSelectedClipSourceAudioTrackVolumeChange, + onSelectedClipSourceAudioTrackNormalizeChange, + embeddedSourcePreviewGain, + getSourceTrackPreviewGain, + }; +} + From 3df4ca873adb85677713fdaf07cde0294fae2bb2 Mon Sep 17 00:00:00 2001 From: Alan Trebugeais Date: Sat, 9 May 2026 15:13:26 +0200 Subject: [PATCH 07/25] add: audio fixed in the wrong layer and stuff --- src/components/video-editor/VideoEditor.tsx | 104 +++++--------- .../video-editor/audio/useAudioPreviewSync.ts | 6 +- .../audio/useClipAudioSettingsController.ts | 3 +- .../audio/useSourceAudioTrackSettings.ts | 15 +++ .../video-editor/audio/useVideoEditorAudio.ts | 127 ++++++++++++++++++ src/components/video-editor/timeline/Item.tsx | 6 + .../video-editor/timeline/TimelineEditor.tsx | 32 +---- .../components/viewport/TimelineCanvas.tsx | 48 ++++--- .../components/waveform/AudioWaveform.tsx | 10 +- 9 files changed, 237 insertions(+), 114 deletions(-) create mode 100644 src/components/video-editor/audio/useVideoEditorAudio.ts diff --git a/src/components/video-editor/VideoEditor.tsx b/src/components/video-editor/VideoEditor.tsx index c7394dda..0930186e 100644 --- a/src/components/video-editor/VideoEditor.tsx +++ b/src/components/video-editor/VideoEditor.tsx @@ -80,7 +80,6 @@ import { canUseInMemoryExportSaveFallback, describeBlockedInMemoryExportSave, } from "@/lib/exporter/exportSavePolicy"; -import { resolveSourceAudioFallbackPaths } from "@/lib/exporter/sourceAudioFallback"; import { matchesShortcut } from "@/lib/shortcuts"; import { cn } from "@/lib/utils"; import { @@ -140,10 +139,7 @@ import { validateProjectData, } from "./projectPersistence"; import { SettingsPanel } from "./SettingsPanel"; -import { useAudioPreviewSync } from "./audio/useAudioPreviewSync"; -import { useClipAudioSettingsController } from "./audio/useClipAudioSettingsController"; -import { useSourceAudioFallback } from "./audio/useSourceAudioFallback"; -import { getActiveClipIdAtSourceTime, isClipMutedById } from "./audio/clipAudio"; +import { useVideoEditorAudio } from "./audio/useVideoEditorAudio"; import { APP_HEADER_ICON_BUTTON_CLASS, DiscordLinkButton, @@ -1768,38 +1764,6 @@ export default function VideoEditor() { () => videoSourcePath ?? (videoPath ? fromFileUrl(videoPath) : null), [videoPath, videoSourcePath], ); - const { sourceAudioFallbackPaths, sourceAudioFallbackStartDelayMsByPath } = - useSourceAudioFallback({ - currentSourcePath, - summarizeErrorMessage, - }); - const { hasEmbeddedSourceAudio, externalAudioPaths: previewSourceAudioFallbackPaths } = useMemo( - () => resolveSourceAudioFallbackPaths(currentSourcePath, sourceAudioFallbackPaths), - [currentSourcePath, sourceAudioFallbackPaths], - ); - const shouldMutePreviewVideo = - !hasEmbeddedSourceAudio && previewSourceAudioFallbackPaths.length > 0; - const activeClipIdAtCurrentTime = useMemo( - () => getActiveClipIdAtSourceTime(currentTime, clipRegions), - [clipRegions, currentTime], - ); - const { - sourceAudioTrackMeta, - activeSourceAudioTrackSettings, - selectedClipSourceAudioTrackSettings, - onSourceAudioTracksMetaChange, - onSelectedClipSourceAudioTrackVolumeChange, - onSelectedClipSourceAudioTrackNormalizeChange, - embeddedSourcePreviewGain, - getSourceTrackPreviewGain, - } = useClipAudioSettingsController({ - selectedClipId, - activeClipId: activeClipIdAtCurrentTime, - }); - const isCurrentClipMuted = useMemo(() => { - return isClipMutedById(activeClipIdAtCurrentTime, clipRegions); - }, [activeClipIdAtCurrentTime, clipRegions]); - const projectDisplayName = useMemo(() => { const fileName = currentProjectPath?.split(/[\\/]/).pop() ?? @@ -3345,6 +3309,25 @@ export default function VideoEditor() { } return result; }, [clipRegions, speedRegions]); + const audio = useVideoEditorAudio({ + currentSourcePath, + selectedClipId, + clipRegions, + audioRegions, + effectiveSpeedRegions, + currentTime, + timelineTime: timelinePlayheadTime, + duration, + isPlaying, + previewVolume, + summarizeErrorMessage, + onSourceFallbackLoadError: (error) => { + toast.warning( + `Could not load companion audio source: ${summarizeErrorMessage(getErrorMessage(error))}`, + { duration: 10000 }, + ); + }, + }); function togglePlayPause() { const playback = videoPlaybackRef.current; @@ -4061,25 +4044,6 @@ export default function VideoEditor() { } }, [selectedAudioId, audioRegions]); - useAudioPreviewSync({ - audioRegions, - previewVolume, - isPlaying, - currentTime, - duration, - effectiveSpeedRegions, - previewSourceAudioFallbackPaths, - sourceAudioFallbackStartDelayMsByPath, - isCurrentClipMuted, - getSourceTrackPreviewGain, - onSourceFallbackLoadError: (error) => { - toast.warning( - `Could not load companion audio source: ${summarizeErrorMessage(getErrorMessage(error))}`, - { duration: 10000 }, - ); - }, - }); - const showExportSuccessToast = useCallback((filePath: string) => { toast.success(`Exported successfully to ${filePath}`, { action: { @@ -4404,8 +4368,9 @@ export default function VideoEditor() { cursorSway, frame, audioRegions, - sourceAudioFallbackPaths, - sourceAudioFallbackStartDelayMsByPath, + sourceAudioFallbackPaths: audio.sourceAudioFallbackPaths, + sourceAudioFallbackStartDelayMsByPath: + audio.sourceAudioFallbackStartDelayMsByPath, previewWidth, previewHeight, onProgress: (progress: ExportProgress) => { @@ -4652,8 +4617,8 @@ export default function VideoEditor() { cursorClickBounceDuration, cursorSway, audioRegions, - sourceAudioFallbackPaths, - sourceAudioFallbackStartDelayMsByPath, + audio.sourceAudioFallbackPaths, + audio.sourceAudioFallbackStartDelayMsByPath, exportEncodingMode, exportBackendPreference, exportPipelineModel, @@ -5752,13 +5717,13 @@ export default function VideoEditor() { hasClipSourceAudio={hasClipSourceAudio} showClipSourceAudioTrack={showClipSourceAudioTrack} onShowClipSourceAudioTrackChange={setShowClipSourceAudioTrack} - sourceAudioTrackMeta={sourceAudioTrackMeta} - sourceAudioTrackSettings={selectedClipSourceAudioTrackSettings} + sourceAudioTrackMeta={audio.sourceAudioTrackMeta} + sourceAudioTrackSettings={audio.selectedClipSourceAudioTrackSettings} onSourceAudioTrackVolumeChange={ - onSelectedClipSourceAudioTrackVolumeChange + audio.onSelectedClipSourceAudioTrackVolumeChange } onSourceAudioTrackNormalizeChange={ - onSelectedClipSourceAudioTrackNormalizeChange + audio.onSelectedClipSourceAudioTrackNormalizeChange } selectedAudioId={selectedAudioId} selectedAudioVolume={ @@ -6065,13 +6030,13 @@ export default function VideoEditor() { } cursorSway={cursorSway} volume={ - shouldMutePreviewVideo || isCurrentClipMuted + audio.isCurrentClipMuted ? 0 : Math.max( 0, Math.min( 1, - previewVolume * embeddedSourcePreviewGain, + previewVolume * audio.embeddedSourcePreviewGain, ), ) } @@ -6350,7 +6315,10 @@ export default function VideoEditor() { onSelectAnnotation={handleSelectAnnotation} aspectRatio={aspectRatio} showSourceAudioTrack={showClipSourceAudioTrack} - sourceAudioTrackSettings={activeSourceAudioTrackSettings} + sourceAudioTrackSettings={audio.activeSourceAudioTrackSettings} + getSourceAudioTrackSettingsForClip={ + audio.getSourceAudioTrackSettingsForClip + } onSourceAudioAvailabilityChange={(available) => { setHasClipSourceAudio(available); if (!available) { @@ -6358,7 +6326,7 @@ export default function VideoEditor() { } }} onSourceAudioTracksMetaChange={(tracks) => { - onSourceAudioTracksMetaChange(tracks); + audio.onSourceAudioTracksMetaChange(tracks); }} /> diff --git a/src/components/video-editor/audio/useAudioPreviewSync.ts b/src/components/video-editor/audio/useAudioPreviewSync.ts index 620ae2e6..b5668194 100644 --- a/src/components/video-editor/audio/useAudioPreviewSync.ts +++ b/src/components/video-editor/audio/useAudioPreviewSync.ts @@ -19,6 +19,7 @@ interface UseAudioPreviewSyncParams { previewVolume: number; isPlaying: boolean; currentTime: number; + timelineTime: number; duration: number; effectiveSpeedRegions: SpeedRegion[]; previewSourceAudioFallbackPaths: string[]; @@ -33,6 +34,7 @@ export function useAudioPreviewSync({ previewVolume, isPlaying, currentTime, + timelineTime, duration, effectiveSpeedRegions, previewSourceAudioFallbackPaths, @@ -219,7 +221,7 @@ export function useAudioPreviewSync({ }, []); useEffect(() => { - const currentTimeMs = currentTime * 1000; + const currentTimeMs = timelineTime * 1000; const activeSpeedRegion = effectiveSpeedRegions.find( (region) => currentTimeMs >= region.startMs && currentTimeMs < region.endMs, ); @@ -252,7 +254,7 @@ export function useAudioPreviewSync({ audio.pause(); } } - }, [audioRegions, currentTime, effectiveSpeedRegions, isPlaying]); + }, [audioRegions, timelineTime, effectiveSpeedRegions, isPlaying]); useEffect(() => { if (previewSourceAudioFallbackPaths.length === 0) { diff --git a/src/components/video-editor/audio/useClipAudioSettingsController.ts b/src/components/video-editor/audio/useClipAudioSettingsController.ts index f6d69881..1873fb52 100644 --- a/src/components/video-editor/audio/useClipAudioSettingsController.ts +++ b/src/components/video-editor/audio/useClipAudioSettingsController.ts @@ -18,6 +18,7 @@ export function useClipAudioSettingsController({ sourceAudioTrackMeta, activeSourceAudioTrackSettings, selectedClipSourceAudioTrackSettings, + getSourceAudioTrackSettingsForClip, onSourceAudioTracksMetaChange, onSelectedClipSourceAudioTrackVolumeChange, onSelectedClipSourceAudioTrackNormalizeChange, @@ -49,6 +50,7 @@ export function useClipAudioSettingsController({ sourceAudioTrackMeta, activeSourceAudioTrackSettings, selectedClipSourceAudioTrackSettings, + getSourceAudioTrackSettingsForClip, onSourceAudioTracksMetaChange, onSelectedClipSourceAudioTrackVolumeChange, onSelectedClipSourceAudioTrackNormalizeChange, @@ -56,4 +58,3 @@ export function useClipAudioSettingsController({ getSourceTrackPreviewGain, }; } - diff --git a/src/components/video-editor/audio/useSourceAudioTrackSettings.ts b/src/components/video-editor/audio/useSourceAudioTrackSettings.ts index 27ea0daf..5a8ca053 100644 --- a/src/components/video-editor/audio/useSourceAudioTrackSettings.ts +++ b/src/components/video-editor/audio/useSourceAudioTrackSettings.ts @@ -13,6 +13,7 @@ export interface UseSourceAudioTrackSettingsResult { sourceAudioTrackMeta: SourceAudioTrackMeta; activeSourceAudioTrackSettings: SourceAudioTrackSettings; selectedClipSourceAudioTrackSettings: SourceAudioTrackSettings; + getSourceAudioTrackSettingsForClip: (clipId: string | null) => SourceAudioTrackSettings; onSourceAudioTracksMetaChange: (tracks: SourceAudioTrackMeta) => void; onSelectedClipSourceAudioTrackVolumeChange: (id: string, volume: number) => void; onSelectedClipSourceAudioTrackNormalizeChange: (id: string, normalize: boolean) => void; @@ -61,6 +62,19 @@ export function useSourceAudioTrackSettings({ }); }, []); + const getSourceAudioTrackSettingsForClip = useCallback( + (clipId: string | null): SourceAudioTrackSettings => { + if (!clipId) { + return defaultSourceAudioTrackSettings; + } + return { + ...defaultSourceAudioTrackSettings, + ...(sourceAudioTrackSettingsByClip[clipId] ?? {}), + }; + }, + [defaultSourceAudioTrackSettings, sourceAudioTrackSettingsByClip], + ); + const onSelectedClipSourceAudioTrackVolumeChange = useCallback( (id: string, volume: number) => { if (!selectedClipId) return; @@ -105,6 +119,7 @@ export function useSourceAudioTrackSettings({ sourceAudioTrackMeta, activeSourceAudioTrackSettings, selectedClipSourceAudioTrackSettings, + getSourceAudioTrackSettingsForClip, onSourceAudioTracksMetaChange, onSelectedClipSourceAudioTrackVolumeChange, onSelectedClipSourceAudioTrackNormalizeChange, diff --git a/src/components/video-editor/audio/useVideoEditorAudio.ts b/src/components/video-editor/audio/useVideoEditorAudio.ts new file mode 100644 index 00000000..02de6a23 --- /dev/null +++ b/src/components/video-editor/audio/useVideoEditorAudio.ts @@ -0,0 +1,127 @@ +import { useMemo } from "react"; +import { resolveSourceAudioFallbackPaths } from "@/lib/exporter/sourceAudioFallback"; +import type { AudioRegion, ClipRegion, SpeedRegion } from "../types"; +import { getActiveClipIdAtSourceTime, isClipMutedById } from "./clipAudio"; +import { useAudioPreviewSync } from "./useAudioPreviewSync"; +import { useClipAudioSettingsController } from "./useClipAudioSettingsController"; +import { useSourceAudioFallback } from "./useSourceAudioFallback"; + +function extractLocalPathFromMediaServerUrl(input: string | null | undefined): string | null { + if (!input) return null; + try { + const url = new URL(input); + const isLocalMediaServer = + (url.protocol === "http:" || url.protocol === "https:") && + (url.hostname === "127.0.0.1" || url.hostname === "localhost") && + url.pathname === "/video"; + if (!isLocalMediaServer) return null; + return url.searchParams.get("path"); + } catch { + return null; + } +} + +interface UseVideoEditorAudioParams { + currentSourcePath: string | null; + selectedClipId: string | null; + clipRegions: ClipRegion[]; + audioRegions: AudioRegion[]; + effectiveSpeedRegions: SpeedRegion[]; + currentTime: number; + timelineTime: number; + duration: number; + isPlaying: boolean; + previewVolume: number; + summarizeErrorMessage: (message: string) => string; + onSourceFallbackLoadError: (error: unknown) => void; +} + +export function useVideoEditorAudio({ + currentSourcePath, + selectedClipId, + clipRegions, + audioRegions, + effectiveSpeedRegions, + currentTime, + timelineTime, + duration, + isPlaying, + previewVolume, + summarizeErrorMessage, + onSourceFallbackLoadError, +}: UseVideoEditorAudioParams) { + const fallbackLookupSourcePath = useMemo( + () => extractLocalPathFromMediaServerUrl(currentSourcePath) ?? currentSourcePath, + [currentSourcePath], + ); + + const { sourceAudioFallbackPaths, sourceAudioFallbackStartDelayMsByPath } = + useSourceAudioFallback({ + currentSourcePath: fallbackLookupSourcePath, + summarizeErrorMessage, + }); + + const { hasEmbeddedSourceAudio, externalAudioPaths: previewSourceAudioFallbackPaths } = useMemo( + () => resolveSourceAudioFallbackPaths(currentSourcePath, sourceAudioFallbackPaths), + [currentSourcePath, sourceAudioFallbackPaths], + ); + const shouldMutePreviewVideo = + !hasEmbeddedSourceAudio && previewSourceAudioFallbackPaths.length > 0; + + const activeClipIdAtCurrentTime = useMemo( + () => getActiveClipIdAtSourceTime(currentTime, clipRegions), + [clipRegions, currentTime], + ); + const isCurrentClipMuted = useMemo( + () => isClipMutedById(activeClipIdAtCurrentTime, clipRegions), + [activeClipIdAtCurrentTime, clipRegions], + ); + + const { + sourceAudioTrackMeta, + activeSourceAudioTrackSettings, + selectedClipSourceAudioTrackSettings, + getSourceAudioTrackSettingsForClip, + onSourceAudioTracksMetaChange, + onSelectedClipSourceAudioTrackVolumeChange, + onSelectedClipSourceAudioTrackNormalizeChange, + embeddedSourcePreviewGain, + getSourceTrackPreviewGain, + } = useClipAudioSettingsController({ + selectedClipId, + activeClipId: activeClipIdAtCurrentTime, + }); + + useAudioPreviewSync({ + audioRegions, + previewVolume, + isPlaying, + currentTime, + timelineTime, + duration, + effectiveSpeedRegions, + previewSourceAudioFallbackPaths, + sourceAudioFallbackStartDelayMsByPath, + isCurrentClipMuted, + getSourceTrackPreviewGain, + onSourceFallbackLoadError, + }); + + return { + sourceAudioFallbackPaths, + sourceAudioFallbackStartDelayMsByPath, + previewSourceAudioFallbackPaths, + shouldMutePreviewVideo, + activeClipIdAtCurrentTime, + isCurrentClipMuted, + sourceAudioTrackMeta, + activeSourceAudioTrackSettings, + selectedClipSourceAudioTrackSettings, + getSourceAudioTrackSettingsForClip, + onSourceAudioTracksMetaChange, + onSelectedClipSourceAudioTrackVolumeChange, + onSelectedClipSourceAudioTrackNormalizeChange, + embeddedSourcePreviewGain, + getSourceTrackPreviewGain, + }; +} diff --git a/src/components/video-editor/timeline/Item.tsx b/src/components/video-editor/timeline/Item.tsx index e633e52e..3af2ad4d 100644 --- a/src/components/video-editor/timeline/Item.tsx +++ b/src/components/video-editor/timeline/Item.tsx @@ -29,6 +29,8 @@ interface ItemProps { speedValue?: number; waveformPeaks?: AudioPeaksData | null; waveformSegmentSpan?: Span; + waveformGain?: number; + waveformNormalize?: boolean; variant?: "zoom" | "trim" | "clip" | "annotation" | "speed" | "audio"; } @@ -65,6 +67,8 @@ export default function Item({ speedValue, waveformPeaks = null, waveformSegmentSpan, + waveformGain = 1, + waveformNormalize = false, variant = "zoom", children, }: ItemProps) { @@ -161,6 +165,8 @@ export default function Item({ peaks={waveformPeaks} segmentStartMs={waveformSegmentSpan?.start ?? span.start} segmentEndMs={waveformSegmentSpan?.end ?? span.end} + gain={waveformGain} + normalize={waveformNormalize} className="absolute inset-0 w-full h-full pointer-events-none opacity-45" /> )} diff --git a/src/components/video-editor/timeline/TimelineEditor.tsx b/src/components/video-editor/timeline/TimelineEditor.tsx index 6fc22499..8fb8a540 100644 --- a/src/components/video-editor/timeline/TimelineEditor.tsx +++ b/src/components/video-editor/timeline/TimelineEditor.tsx @@ -85,6 +85,9 @@ export interface TimelineEditorProps { showSourceAudioTrack?: boolean; onSourceAudioAvailabilityChange?: (available: boolean) => void; sourceAudioTrackSettings?: Record; + getSourceAudioTrackSettingsForClip?: ( + clipId: string | null, + ) => Record; onSourceAudioTracksMetaChange?: (tracks: Array<{ id: string; label: string }>) => void; } @@ -172,6 +175,7 @@ const TimelineEditor = forwardRef( showSourceAudioTrack = false, onSourceAudioAvailabilityChange, sourceAudioTrackSettings = {}, + getSourceAudioTrackSettingsForClip, onSourceAudioTracksMetaChange, }, ref, @@ -296,30 +300,7 @@ const TimelineEditor = forwardRef( useEffect(() => { onSourceAudioTracksMetaChange?.(sourceAudioTracks.map((t) => ({ id: t.id, label: t.label }))); }, [onSourceAudioTracksMetaChange, sourceAudioTracks]); - const displaySourceAudioTracks = useMemo(() => { - return sourceAudioTracks.map((track) => { - const settings = sourceAudioTrackSettings[track.id] ?? { volume: 1, normalize: false }; - const volume = Math.max(0, Math.min(2, settings.volume)); - const normalize = settings.normalize; - const input = track.peaks.peaks; - const adjusted = new Float32Array(input.length); - for (let i = 0; i < input.length; i++) { - let amp = input[i]; - if (normalize) { - amp = Math.sqrt(amp); - } - adjusted[i] = Math.max(0, Math.min(1, amp * volume)); - } - return { - id: track.id, - label: track.label, - peaks: { - durationMs: track.peaks.durationMs, - peaks: adjusted, - } satisfies AudioPeaksData, - }; - }); - }, [sourceAudioTrackSettings, sourceAudioTracks]); + void sourceAudioTrackSettings; useEffect(() => { onSourceAudioAvailabilityChange?.(sourceAudioTracks.length > 0); }, [onSourceAudioAvailabilityChange, sourceAudioTracks.length]); @@ -565,7 +546,8 @@ const TimelineEditor = forwardRef( selectAllBlocksActive={selectAllBlocksActive} onClearBlockSelection={clearSelectedBlocks} keyframes={keyframes} - sourceAudioTracks={displaySourceAudioTracks} + sourceAudioTracks={sourceAudioTracks} + getSourceAudioTrackSettingsForClip={getSourceAudioTrackSettingsForClip} showSourceAudioTrack={showSourceAudioTrack} liveSpanPreviewById={liveZoomPreview.previewSpans} liveHiddenItemIds={Array.from(liveZoomPreview.hiddenZoomIds)} diff --git a/src/components/video-editor/timeline/components/viewport/TimelineCanvas.tsx b/src/components/video-editor/timeline/components/viewport/TimelineCanvas.tsx index 94d4cd88..12013a27 100644 --- a/src/components/video-editor/timeline/components/viewport/TimelineCanvas.tsx +++ b/src/components/video-editor/timeline/components/viewport/TimelineCanvas.tsx @@ -58,6 +58,9 @@ interface TimelineCanvasProps { onClearBlockSelection?: () => void; keyframes?: { id: string; time: number }[]; sourceAudioTracks?: Array<{ id: string; label: string; peaks: AudioPeaksData }>; + getSourceAudioTrackSettingsForClip?: ( + clipId: string | null, + ) => Record; showSourceAudioTrack?: boolean; liveSpanPreviewById?: Record; liveHiddenItemIds?: string[]; @@ -222,6 +225,9 @@ interface TimelineCanvasRowsProps { onSelectAnnotation?: (id: string | null) => void; onSelectAudio?: (id: string | null) => void; sourceAudioTracks?: Array<{ id: string; label: string; peaks: AudioPeaksData }>; + getSourceAudioTrackSettingsForClip?: ( + clipId: string | null, + ) => Record; showSourceAudioTrack?: boolean; liveSpanPreviewById?: Record; liveHiddenItemIds?: string[]; @@ -285,6 +291,7 @@ const TimelineCanvasRows = memo(function TimelineCanvasRows({ onSelectAnnotation, onSelectAudio, sourceAudioTracks = [], + getSourceAudioTrackSettingsForClip, showSourceAudioTrack = false, liveSpanPreviewById, liveHiddenItemIds, @@ -371,22 +378,29 @@ const TimelineCanvasRows = memo(function TimelineCanvasRows({ {showSourceAudioTrack && sourceAudioTracks.map((track) => ( - {clipItems.map((item) => ( - onSelectClip?.(item.id)} - variant="audio" - waveformPeaks={track.peaks} - waveformSegmentSpan={liveSpanPreviewById?.[item.id] ?? item.span} - > - {track.label} - - ))} + {clipItems.map((item) => { + const settings = getSourceAudioTrackSettingsForClip?.(item.id)?.[ + track.id + ] ?? { volume: 1, normalize: false }; + return ( + onSelectClip?.(item.id)} + variant="audio" + waveformPeaks={track.peaks} + waveformSegmentSpan={liveSpanPreviewById?.[item.id] ?? item.span} + waveformGain={Math.max(0, Math.min(2, settings.volume))} + waveformNormalize={Boolean(settings.normalize)} + > + {track.label} + + ); + })} ))} @@ -497,6 +511,7 @@ export default function TimelineCanvas({ onClearBlockSelection, keyframes = [], sourceAudioTracks = [], + getSourceAudioTrackSettingsForClip, showSourceAudioTrack = false, liveSpanPreviewById, liveHiddenItemIds, @@ -730,6 +745,7 @@ export default function TimelineCanvas({ onSelectAnnotation={onSelectAnnotation} onSelectAudio={onSelectAudio} sourceAudioTracks={sourceAudioTracks} + getSourceAudioTrackSettingsForClip={getSourceAudioTrackSettingsForClip} showSourceAudioTrack={showSourceAudioTrack} liveSpanPreviewById={liveSpanPreviewById} liveHiddenItemIds={liveHiddenItemIds} diff --git a/src/components/video-editor/timeline/components/waveform/AudioWaveform.tsx b/src/components/video-editor/timeline/components/waveform/AudioWaveform.tsx index 4b8bf510..ae06fd49 100644 --- a/src/components/video-editor/timeline/components/waveform/AudioWaveform.tsx +++ b/src/components/video-editor/timeline/components/waveform/AudioWaveform.tsx @@ -6,6 +6,8 @@ interface AudioWaveformProps { peaks: AudioPeaksData; segmentStartMs?: number; segmentEndMs?: number; + gain?: number; + normalize?: boolean; className?: string; } @@ -18,6 +20,8 @@ function AudioWaveformComponent({ peaks, segmentStartMs, segmentEndMs, + gain = 1, + normalize = false, className, }: AudioWaveformProps) { const canvasRef = useRef(null); @@ -90,7 +94,9 @@ function AudioWaveformComponent({ const leftIndex = Math.floor(exactIndex); const rightIndex = Math.min(peakData.length - 1, leftIndex + 1); const mix = exactIndex - leftIndex; - const amplitude = peakData[leftIndex] * (1 - mix) + peakData[rightIndex] * mix; + let amplitude = peakData[leftIndex] * (1 - mix) + peakData[rightIndex] * mix; + if (normalize) amplitude = Math.sqrt(Math.max(0, amplitude)); + amplitude = Math.max(0, Math.min(1, amplitude * gain)); const barHeight = amplitude * midY * 0.85; ctx.moveTo(px, midY - barHeight); @@ -103,7 +109,7 @@ function AudioWaveformComponent({ }; rafId = requestAnimationFrame(draw); return () => cancelAnimationFrame(rafId); - }, [peaks, range.start, range.end, resizeKey, segmentStartMs, segmentEndMs]); + }, [gain, normalize, peaks, range.start, range.end, resizeKey, segmentStartMs, segmentEndMs]); return ( Date: Sat, 9 May 2026 15:28:28 +0200 Subject: [PATCH 08/25] you can now seperate clip audio for each clip --- src/components/video-editor/SettingsPanel.tsx | 75 ++++++++----------- src/components/video-editor/VideoEditor.tsx | 40 ++++++---- .../components/viewport/TimelineCanvas.tsx | 2 +- .../timeline/core/timelineTypes.ts | 1 + .../timeline/model/timelineModel.ts | 1 + src/components/video-editor/types.ts | 1 + src/i18n/locales/en/settings.json | 8 ++ src/i18n/locales/es/settings.json | 8 ++ src/i18n/locales/fr/settings.json | 8 ++ src/i18n/locales/ko/settings.json | 2 + src/i18n/locales/nl/settings.json | 2 + src/i18n/locales/pt-BR/settings.json | 2 + src/i18n/locales/ru/settings.json | 2 + src/i18n/locales/zh-CN/settings.json | 2 + src/i18n/locales/zh-TW/settings.json | 2 + 15 files changed, 97 insertions(+), 59 deletions(-) diff --git a/src/components/video-editor/SettingsPanel.tsx b/src/components/video-editor/SettingsPanel.tsx index 657398a2..7e05203d 100644 --- a/src/components/video-editor/SettingsPanel.tsx +++ b/src/components/video-editor/SettingsPanel.tsx @@ -469,11 +469,11 @@ interface SettingsPanelProps { selectedClipId?: string | null; selectedClipSpeed?: number | null; selectedClipMuted?: boolean | null; + selectedClipShowSourceAudio?: boolean | null; hasClipSourceAudio?: boolean; - showClipSourceAudioTrack?: boolean; onClipSpeedChange?: (speed: number) => void; onClipMutedChange?: (muted: boolean) => void; - onShowClipSourceAudioTrackChange?: (show: boolean) => void; + onClipShowSourceAudioChange?: (show: boolean) => void; sourceAudioTrackMeta?: Array<{ id: string; label: string }>; sourceAudioTrackSettings?: Record; onSourceAudioTrackVolumeChange?: (id: string, volume: number) => void; @@ -869,11 +869,11 @@ export function SettingsPanel({ selectedClipId, selectedClipSpeed, selectedClipMuted, + selectedClipShowSourceAudio = false, hasClipSourceAudio = false, - showClipSourceAudioTrack = false, onClipSpeedChange, onClipMutedChange, - onShowClipSourceAudioTrackChange, + onClipShowSourceAudioChange, sourceAudioTrackMeta = [], sourceAudioTrackSettings = {}, onSourceAudioTrackVolumeChange, @@ -3059,42 +3059,7 @@ export function SettingsPanel({ )} -
- - {tSettings("clip.muteAudio", "Mute Audio")} - -
- - {selectedClipMuted - ? tSettings("clip.mutedState", "Muted") - : tSettings("clip.audioOnState", "Audio On")} - - onClipMutedChange?.(!v)} - className="data-[state=checked]:bg-[#06b6d4] scale-75" - /> -
-
- {hasClipSourceAudio && ( -
- - {tSettings("clip.separateSourceAudio", "Separate Clip Audio")} - - onShowClipSourceAudioTrackChange?.(v)} - className="data-[state=checked]:bg-[#06b6d4] scale-75" - /> -
- )} +
{tSettings("speed.label", "Speed")}
@@ -3135,6 +3100,32 @@ export function SettingsPanel({ ); })} + +
+ + {selectedClipMuted + ? tSettings("clip.unmuteAudio", "Unmute Audio") + : tSettings("clip.muteAudio", "Mute Audio")} + + onClipMutedChange?.(!v)} + className="data-[state=checked]:bg-[#06b6d4] scale-75" + /> +
+ {hasClipSourceAudio && ( +
+ + {tSettings("clip.separateClipFromAudio", "Separate clip from audio")} + + onClipShowSourceAudioChange?.(v)} + className="data-[state=checked]:bg-[#06b6d4] scale-75" + /> +
+ )} + {selectedClipId && ( - ) : selectedClipId && hasClipSourceAudio && showClipSourceAudioTrack ? ( + ) : selectedClipId && hasClipSourceAudio && selectedClipShowSourceAudio ? (
{tSettings("audio.sourceTracksTitle", "Clip Source Audio")} diff --git a/src/components/video-editor/VideoEditor.tsx b/src/components/video-editor/VideoEditor.tsx index 0930186e..dacffaa9 100644 --- a/src/components/video-editor/VideoEditor.tsx +++ b/src/components/video-editor/VideoEditor.tsx @@ -665,7 +665,6 @@ export default function VideoEditor() { const [audioRegions, setAudioRegions] = useState([]); const [selectedAudioId, setSelectedAudioId] = useState(null); const [hasClipSourceAudio, setHasClipSourceAudio] = useState(false); - const [showClipSourceAudioTrack, setShowClipSourceAudioTrack] = useState(false); const [autoCaptions, setAutoCaptions] = useState([]); const [autoCaptionSettings, setAutoCaptionSettings] = useState( DEFAULT_AUTO_CAPTION_SETTINGS, @@ -3709,6 +3708,18 @@ export default function VideoEditor() { [selectedClipId], ); + const handleClipShowSourceAudioChange = useCallback( + (showSourceAudio: boolean) => { + if (!selectedClipId) return; + setClipRegions((prev) => + prev.map((clip) => + clip.id === selectedClipId ? { ...clip, showSourceAudio } : clip, + ), + ); + }, + [selectedClipId], + ); + const handleClipDelete = useCallback( (id: string) => { const deletedClip = clipRegions.find((clip) => clip.id === id); @@ -5697,26 +5708,26 @@ export default function VideoEditor() { selectedClipId={selectedClipId} selectedClipSpeed={ selectedClipId - ? (clipRegions.find((c) => c.id === selectedClipId) - ?.speed ?? 1) + ? clipRegions.find((c) => c.id === selectedClipId)?.speed ?? 1 : null } selectedClipMuted={ selectedClipId - ? (clipRegions.find((c) => c.id === selectedClipId) - ?.muted ?? false) + ? clipRegions.find((c) => c.id === selectedClipId)?.muted ?? + false : null } - onClipSpeedChange={(speed) => - selectedClipId && handleClipSpeedChange(speed) - } - onClipMutedChange={(muted) => - selectedClipId && handleClipMutedChange(muted) + selectedClipShowSourceAudio={ + selectedClipId + ? clipRegions.find((c) => c.id === selectedClipId) + ?.showSourceAudio ?? false + : null } + onClipSpeedChange={handleClipSpeedChange} + onClipMutedChange={handleClipMutedChange} + onClipShowSourceAudioChange={handleClipShowSourceAudioChange} onClipDelete={handleClipDelete} hasClipSourceAudio={hasClipSourceAudio} - showClipSourceAudioTrack={showClipSourceAudioTrack} - onShowClipSourceAudioTrackChange={setShowClipSourceAudioTrack} sourceAudioTrackMeta={audio.sourceAudioTrackMeta} sourceAudioTrackSettings={audio.selectedClipSourceAudioTrackSettings} onSourceAudioTrackVolumeChange={ @@ -6314,16 +6325,13 @@ export default function VideoEditor() { selectedAnnotationId={selectedAnnotationId} onSelectAnnotation={handleSelectAnnotation} aspectRatio={aspectRatio} - showSourceAudioTrack={showClipSourceAudioTrack} + showSourceAudioTrack={clipRegions.some((c) => c.showSourceAudio)} sourceAudioTrackSettings={audio.activeSourceAudioTrackSettings} getSourceAudioTrackSettingsForClip={ audio.getSourceAudioTrackSettingsForClip } onSourceAudioAvailabilityChange={(available) => { setHasClipSourceAudio(available); - if (!available) { - setShowClipSourceAudioTrack(false); - } }} onSourceAudioTracksMetaChange={(tracks) => { audio.onSourceAudioTracksMetaChange(tracks); diff --git a/src/components/video-editor/timeline/components/viewport/TimelineCanvas.tsx b/src/components/video-editor/timeline/components/viewport/TimelineCanvas.tsx index 12013a27..a1617779 100644 --- a/src/components/video-editor/timeline/components/viewport/TimelineCanvas.tsx +++ b/src/components/video-editor/timeline/components/viewport/TimelineCanvas.tsx @@ -378,7 +378,7 @@ const TimelineCanvasRows = memo(function TimelineCanvasRows({ {showSourceAudioTrack && sourceAudioTracks.map((track) => ( - {clipItems.map((item) => { + {clipItems.filter(item => item.showSourceAudio).map((item) => { const settings = getSourceAudioTrackSettingsForClip?.(item.id)?.[ track.id ] ?? { volume: 1, normalize: false }; diff --git a/src/components/video-editor/timeline/core/timelineTypes.ts b/src/components/video-editor/timeline/core/timelineTypes.ts index 4a58c969..1fc9a66a 100644 --- a/src/components/video-editor/timeline/core/timelineTypes.ts +++ b/src/components/video-editor/timeline/core/timelineTypes.ts @@ -36,6 +36,7 @@ export interface TimelineRenderItem { zoomDepth?: number; zoomMode?: ZoomMode; speedValue?: number; + showSourceAudio?: boolean; variant: "zoom" | "trim" | "clip" | "annotation" | "speed" | "audio"; } diff --git a/src/components/video-editor/timeline/model/timelineModel.ts b/src/components/video-editor/timeline/model/timelineModel.ts index b36aabc9..600859a5 100644 --- a/src/components/video-editor/timeline/model/timelineModel.ts +++ b/src/components/video-editor/timeline/model/timelineModel.ts @@ -52,6 +52,7 @@ export function buildTimelineItems(params: { rowId: CLIP_ROW_ID, span: { start: region.startMs, end: region.endMs }, label: `Clip ${index + 1}`, + showSourceAudio: region.showSourceAudio, variant: "clip", })); diff --git a/src/components/video-editor/types.ts b/src/components/video-editor/types.ts index 92f84fd6..ec65c44e 100644 --- a/src/components/video-editor/types.ts +++ b/src/components/video-editor/types.ts @@ -169,6 +169,7 @@ export interface ClipRegion { endMs: number; speed: number; muted?: boolean; + showSourceAudio?: boolean; } export function getClipSourceEndMs(clip: ClipRegion): number { diff --git a/src/i18n/locales/en/settings.json b/src/i18n/locales/en/settings.json index 3a2b6a76..7b942498 100644 --- a/src/i18n/locales/en/settings.json +++ b/src/i18n/locales/en/settings.json @@ -20,6 +20,8 @@ "clip": { "title": "Clip", "muteAudio": "Mute Audio", + "unmuteAudio": "Unmute Audio", + "separateClipFromAudio": "Separate clip from audio", "delete": "Delete Clip" }, "effects": { @@ -201,5 +203,11 @@ "exportVideo": "Export {{format}}", "reportBug": "Report Bug", "starOnGithub": "Star on GitHub" + }, + "audio": { + "sourceTracksTitle": "Clip Source Audio", + "volumeTitle": "Volume", + "normalize": "Normalize", + "deleteRegion": "Delete Audio" } } diff --git a/src/i18n/locales/es/settings.json b/src/i18n/locales/es/settings.json index 0c7ca244..75f78d75 100644 --- a/src/i18n/locales/es/settings.json +++ b/src/i18n/locales/es/settings.json @@ -20,6 +20,8 @@ "clip": { "title": "Clip", "muteAudio": "Silenciar audio", + "unmuteAudio": "Activar sonido", + "separateClipFromAudio": "Separar clip del audio", "delete": "Eliminar clip" }, "effects": { @@ -181,5 +183,11 @@ "exportVideo": "Exportar {{format}}", "reportBug": "Reportar error", "starOnGithub": "Estrella en GitHub" + }, + "audio": { + "sourceTracksTitle": "Audio fuente del clip", + "volumeTitle": "Volumen", + "normalize": "Normalizar", + "deleteRegion": "Eliminar audio" } } diff --git a/src/i18n/locales/fr/settings.json b/src/i18n/locales/fr/settings.json index 04804828..241bc269 100644 --- a/src/i18n/locales/fr/settings.json +++ b/src/i18n/locales/fr/settings.json @@ -20,6 +20,8 @@ "clip": { "title": "Clip", "muteAudio": "Couper le son", + "unmuteAudio": "Réactiver le son", + "separateClipFromAudio": "Séparer le clip de l'audio", "delete": "Supprimer le clip" }, "effects": { @@ -181,5 +183,11 @@ "exportVideo": "Exporter en {{format}}", "reportBug": "Signaler un bug", "starOnGithub": "Mettre une étoile sur GitHub" + }, + "audio": { + "sourceTracksTitle": "Audio source du clip", + "volumeTitle": "Volume", + "normalize": "Normaliser", + "deleteRegion": "Supprimer l'audio" } } diff --git a/src/i18n/locales/ko/settings.json b/src/i18n/locales/ko/settings.json index 8c32549a..0adaa6c7 100644 --- a/src/i18n/locales/ko/settings.json +++ b/src/i18n/locales/ko/settings.json @@ -20,6 +20,8 @@ "clip": { "title": "클립", "muteAudio": "오디오 음소거", + "unmuteAudio": "음소거 해제", + "separateClipFromAudio": "클립과 오디오 분리", "delete": "클립 삭제" }, "effects": { diff --git a/src/i18n/locales/nl/settings.json b/src/i18n/locales/nl/settings.json index 1e7bb77d..00d78905 100644 --- a/src/i18n/locales/nl/settings.json +++ b/src/i18n/locales/nl/settings.json @@ -20,6 +20,8 @@ "clip": { "title": "Clip", "muteAudio": "Audio dempen", + "unmuteAudio": "Geluid inschakelen", + "separateClipFromAudio": "Clip scheiden van audio", "delete": "Clip verwijderen" }, "effects": { diff --git a/src/i18n/locales/pt-BR/settings.json b/src/i18n/locales/pt-BR/settings.json index 58783df0..810d7f91 100644 --- a/src/i18n/locales/pt-BR/settings.json +++ b/src/i18n/locales/pt-BR/settings.json @@ -20,6 +20,8 @@ "clip": { "title": "Clipe", "muteAudio": "Silenciar áudio", + "unmuteAudio": "Ativar som", + "separateClipFromAudio": "Separar clipe do áudio", "delete": "Excluir clipe" }, "effects": { diff --git a/src/i18n/locales/ru/settings.json b/src/i18n/locales/ru/settings.json index 88e48ec5..6a2f052d 100644 --- a/src/i18n/locales/ru/settings.json +++ b/src/i18n/locales/ru/settings.json @@ -20,6 +20,8 @@ "clip": { "title": "Клип", "muteAudio": "Выключить звук", + "unmuteAudio": "Включить звук", + "separateClipFromAudio": "Отделить клип от аудио", "delete": "Удалить" }, "effects": { diff --git a/src/i18n/locales/zh-CN/settings.json b/src/i18n/locales/zh-CN/settings.json index 5f5b277a..f895653f 100644 --- a/src/i18n/locales/zh-CN/settings.json +++ b/src/i18n/locales/zh-CN/settings.json @@ -20,6 +20,8 @@ "clip": { "title": "片段", "muteAudio": "静音音频", + "unmuteAudio": "取消静音", + "separateClipFromAudio": "将剪辑与音频分离", "delete": "删除片段" }, "effects": { diff --git a/src/i18n/locales/zh-TW/settings.json b/src/i18n/locales/zh-TW/settings.json index bf2ffc9c..fd9ff235 100644 --- a/src/i18n/locales/zh-TW/settings.json +++ b/src/i18n/locales/zh-TW/settings.json @@ -20,6 +20,8 @@ "clip": { "title": "片段", "muteAudio": "靜音", + "unmuteAudio": "取消靜音", + "separateClipFromAudio": "將片段與音訊分離", "delete": "刪除片段" }, "effects": { From f4af06d430c653d1b360bec3efb52cfa6c7ab09b Mon Sep 17 00:00:00 2001 From: Alan Trebugeais Date: Sat, 9 May 2026 18:07:28 +0200 Subject: [PATCH 09/25] add: project persistence of audio settings --- electron/ipc/recording/diagnostics.ts | 9 +- electron/ipc/recording/mac.ts | 6 - electron/ipc/recording/windows.ts | 26 +- .../video-editor/AnnotationSettingsPanel.tsx | 25 +- src/components/video-editor/SettingsPanel.tsx | 277 ++++++++++-------- src/components/video-editor/VideoEditor.tsx | 23 ++ .../video-editor/audio/useAudioPreviewSync.ts | 21 +- .../audio/useClipAudioSettingsController.ts | 43 ++- .../audio/useSourceAudioTrackSettings.ts | 21 +- .../video-editor/audio/useVideoEditorAudio.ts | 31 +- .../video-editor/projectPersistence.ts | 17 ++ .../video-editor/timeline/TimelineEditor.tsx | 26 +- src/components/video-editor/types.ts | 8 + src/i18n/locales/en/settings.json | 3 + src/i18n/locales/es/settings.json | 3 + src/i18n/locales/fr/settings.json | 3 + src/i18n/locales/zh-CN/settings.json | 11 +- 17 files changed, 366 insertions(+), 187 deletions(-) diff --git a/electron/ipc/recording/diagnostics.ts b/electron/ipc/recording/diagnostics.ts index b8fb6860..f097cea9 100644 --- a/electron/ipc/recording/diagnostics.ts +++ b/electron/ipc/recording/diagnostics.ts @@ -499,20 +499,21 @@ export async function getCompanionAudioFallbackInfo(videoPath: string) { let paths: string[]; if (await hasEmbeddedAudioStream(videoPath)) { - const microphoneCompanionPaths = Array.from( + const companionPaths = Array.from( new Set( companionCandidates.flatMap((candidate) => candidate.usablePaths.filter( - (companionPath) => companionPath === candidate.micPath, + (companionPath) => + companionPath === candidate.micPath || companionPath === candidate.systemPath, ), ), ), ); - if (microphoneCompanionPaths.length === 0) { + if (companionPaths.length === 0) { return { paths: [], startDelayMsByPath: {} }; } - paths = [videoPath, ...microphoneCompanionPaths]; + paths = [videoPath, ...companionPaths]; } else { paths = Array.from( new Set(companionCandidates.flatMap((candidate) => candidate.usablePaths)), diff --git a/electron/ipc/recording/mac.ts b/electron/ipc/recording/mac.ts index 769d22ca..1227b0af 100644 --- a/electron/ipc/recording/mac.ts +++ b/electron/ipc/recording/mac.ts @@ -277,12 +277,6 @@ export async function muxNativeMacRecordingWithAudio( await moveFileWithOverwrite(mixedOutputPath, videoPath); console.log("[mux] Successfully muxed audio into video:", videoPath); - - for (const audioPath of [systemAudioPath, microphonePath]) { - if (audioPath) { - await fs.rm(audioPath, { force: true }).catch(() => undefined); - } - } } export function attachNativeCaptureLifecycle(process: ChildProcessWithoutNullStreams) { diff --git a/electron/ipc/recording/windows.ts b/electron/ipc/recording/windows.ts index 6fc6c035..26e15b07 100644 --- a/electron/ipc/recording/windows.ts +++ b/electron/ipc/recording/windows.ts @@ -474,30 +474,6 @@ export async function muxNativeWindowsVideoWithAudio( throw error; } - if (keepAudioSidecars) { - console.log( - `[mux-win] Keeping native audio sidecars because ${RECORDING_AUDIO_SIDECAR_DEBUG_ENV} is enabled`, - ); - return { - muxed: true, - videoDurationSeconds: videoDuration, - muxTimeoutMs, - audioInputs, - audio, - outputPath: videoPath, - keptAudioSidecars: true, - }; - } - - for (const audioPath of [systemAudioPath, micAudioPath]) { - if (audioPath) { - await Promise.all([ - fs.rm(audioPath, { force: true }).catch(() => undefined), - fs.rm(`${audioPath}.json`, { force: true }).catch(() => undefined), - ]); - } - } - return { muxed: true, videoDurationSeconds: videoDuration, @@ -505,6 +481,6 @@ export async function muxNativeWindowsVideoWithAudio( audioInputs, audio, outputPath: videoPath, - keptAudioSidecars: false, + keptAudioSidecars: true, }; } diff --git a/src/components/video-editor/AnnotationSettingsPanel.tsx b/src/components/video-editor/AnnotationSettingsPanel.tsx index a0370a5d..be475809 100644 --- a/src/components/video-editor/AnnotationSettingsPanel.tsx +++ b/src/components/video-editor/AnnotationSettingsPanel.tsx @@ -139,7 +139,8 @@ export function AnnotationSettingsPanel({ }; return ( -
+
+
@@ -772,16 +773,6 @@ export function AnnotationSettingsPanel({ - -
@@ -796,6 +787,18 @@ export function AnnotationSettingsPanel({
+
+ +
+
); } diff --git a/src/components/video-editor/SettingsPanel.tsx b/src/components/video-editor/SettingsPanel.tsx index 7e05203d..88dd2659 100644 --- a/src/components/video-editor/SettingsPanel.tsx +++ b/src/components/video-editor/SettingsPanel.tsx @@ -3049,6 +3049,28 @@ export function SettingsPanel({ ); + const audioSectionContent = ( +
+
+ {tSettings("audio.volumeTitle", "Audio")} + + {Math.round((selectedAudioVolume ?? 1) * 100)}% + +
+ onAudioVolumeChange?.(v)} + formatValue={(v) => `${Math.round(v * 100)}%`} + parseInput={(text) => parseFloat(text.replace(/%$/, "")) / 100} + /> +
+ ); + const clipSectionContent = (
@@ -3101,44 +3123,88 @@ export function SettingsPanel({ })}
-
- - {selectedClipMuted - ? tSettings("clip.unmuteAudio", "Unmute Audio") - : tSettings("clip.muteAudio", "Mute Audio")} - - onClipMutedChange?.(!v)} - className="data-[state=checked]:bg-[#06b6d4] scale-75" - /> -
- {hasClipSourceAudio && ( -
+
+ {tSettings("audio.title", "Audio")} + +
- {tSettings("clip.separateClipFromAudio", "Separate clip from audio")} + {selectedClipMuted + ? tSettings("clip.unmuteAudio", "Unmute Audio") + : tSettings("clip.muteAudio", "Mute Audio")} onClipShowSourceAudioChange?.(v)} + checked={!(selectedClipMuted ?? false)} + onCheckedChange={(v) => onClipMutedChange?.(!v)} className="data-[state=checked]:bg-[#06b6d4] scale-75" />
- )} + {hasClipSourceAudio && ( +
+ + {tSettings("clip.separateClipFromAudio", "Separate clip from audio")} + + onClipShowSourceAudioChange?.(v)} + className="data-[state=checked]:bg-[#06b6d4] scale-75" + /> +
+ )} +
- {selectedClipId && ( - - )} + {selectedClipId && + hasClipSourceAudio && + selectedClipShowSourceAudio && + sourceAudioTrackMeta.length > 0 && ( +
+ {sourceAudioTrackMeta.map((track) => { + const settings = sourceAudioTrackSettings[track.id] ?? { + volume: 1, + normalize: false, + }; + return ( +
+
+ + {track.label} + + + {Math.round(settings.volume * 100)}% + +
+
+ + {tSettings("audio.normalize", "Normalize")} + + + onSourceAudioTrackNormalizeChange?.(track.id, v) + } + className="data-[state=checked]:bg-[#06b6d4] scale-75" + /> +
+ onSourceAudioTrackVolumeChange?.(track.id, v)} + formatValue={(v) => `${Math.round(v * 100)}%`} + parseInput={(text) => + parseFloat(text.replace(/%$/, "")) / 100 + } + /> +
+ ); + })} +
+ )}
); @@ -3151,6 +3217,8 @@ export function SettingsPanel({ return zoomItemSectionContent; case "clip": return clipSectionContent; + case "audio": + return audioSectionContent; case "frame": return sceneSectionContent; case "crop": @@ -3592,94 +3660,69 @@ export function SettingsPanel({
{ + if (activeEffectSection === "clip" && selectedClipId) return false; + if (activeEffectSection === "zoom" && selectedZoomId) return false; + if (activeEffectSection === "audio" && selectedAudioId) return false; + if (selectedAnnotationId) return false; // Annotation editor handles its own but let's see + return true; + })() && "hidden", )} > - {selectedAudioId ? ( -
-
- - {tSettings("audio.volumeTitle", "Audio Volume")} - - - {Math.round((selectedAudioVolume ?? 1) * 100)}% - -
- onAudioVolumeChange?.(v)} - formatValue={(v) => `${Math.round(v * 100)}%`} - parseInput={(text) => parseFloat(text.replace(/%$/, "")) / 100} - /> - -
- ) : selectedClipId && hasClipSourceAudio && selectedClipShowSourceAudio ? ( -
-
- {tSettings("audio.sourceTracksTitle", "Clip Source Audio")} -
- {sourceAudioTrackMeta.map((track) => { - const settings = sourceAudioTrackSettings[track.id] ?? { - volume: 1, - normalize: false, - }; - return ( -
-
- - {track.label} - - - {Math.round(settings.volume * 100)}% - -
-
- - {tSettings("audio.normalize", "Normalize")} - - - onSourceAudioTrackNormalizeChange?.(track.id, v) - } - className="data-[state=checked]:bg-[#2563EB] scale-75" - /> -
- onSourceAudioTrackVolumeChange?.(track.id, v)} - formatValue={(v) => `${Math.round(v * 100)}%`} - parseInput={(text) => parseFloat(text.replace(/%$/, "")) / 100} - /> -
- ); - })} -
- ) : null} + {activeEffectSection === "clip" && selectedClipId && ( + + )} + {activeEffectSection === "zoom" && selectedZoomId && ( + + )} + {activeEffectSection === "audio" && selectedAudioId && ( + + )} + {selectedAnnotationId && ( + + )}
); diff --git a/src/components/video-editor/VideoEditor.tsx b/src/components/video-editor/VideoEditor.tsx index dacffaa9..fbc61a63 100644 --- a/src/components/video-editor/VideoEditor.tsx +++ b/src/components/video-editor/VideoEditor.tsx @@ -197,6 +197,7 @@ import { type ZoomMotionBlurTuning, type ZoomRegion, type ZoomTransitionEasing, + type SourceAudioTrackSettings, } from "./types"; import VideoPlayback, { VideoPlaybackRef } from "./VideoPlayback"; import { @@ -664,6 +665,12 @@ export default function VideoEditor() { const [selectedAnnotationId, setSelectedAnnotationId] = useState(null); const [audioRegions, setAudioRegions] = useState([]); const [selectedAudioId, setSelectedAudioId] = useState(null); + const [sourceAudioTrackSettingsByClip, setSourceAudioTrackSettingsByClip] = useState< + Record + >({}); + const [defaultSourceAudioTrackSettings, setDefaultSourceAudioTrackSettings] = useState< + SourceAudioTrackSettings + >({}); const [hasClipSourceAudio, setHasClipSourceAudio] = useState(false); const [autoCaptions, setAutoCaptions] = useState([]); const [autoCaptionSettings, setAutoCaptionSettings] = useState( @@ -1752,6 +1759,8 @@ export default function VideoEditor() { gifFrameRate: GifFrameRate; gifLoop: boolean; gifSizePreset: GifSizePreset; + sourceAudioTrackSettingsByClip: Record; + defaultSourceAudioTrackSettings: SourceAudioTrackSettings; }>, ) => { return editor; @@ -1853,6 +1862,8 @@ export default function VideoEditor() { gifFrameRate, gifLoop, gifSizePreset, + sourceAudioTrackSettingsByClip, + defaultSourceAudioTrackSettings, }), [ buildPersistedEditorState, @@ -1913,6 +1924,8 @@ export default function VideoEditor() { gifLoop, gifSizePreset, frame, + sourceAudioTrackSettingsByClip, + defaultSourceAudioTrackSettings, ], ); @@ -2098,6 +2111,10 @@ export default function VideoEditor() { setSpeedRegions(normalizedEditor.speedRegions); setAnnotationRegions(normalizedEditor.annotationRegions); setAudioRegions(normalizedEditor.audioRegions); + setSourceAudioTrackSettingsByClip(normalizedEditor.sourceAudioTrackSettingsByClip ?? {}); + setDefaultSourceAudioTrackSettings( + normalizedEditor.defaultSourceAudioTrackSettings ?? {}, + ); setAutoCaptions(normalizedEditor.autoCaptions); setAutoCaptionSettings(normalizedEditor.autoCaptionSettings); setAspectRatio(normalizedEditor.aspectRatio); @@ -3314,6 +3331,10 @@ export default function VideoEditor() { clipRegions, audioRegions, effectiveSpeedRegions, + sourceAudioTrackSettingsByClip, + setSourceAudioTrackSettingsByClip, + defaultSourceAudioTrackSettings, + setDefaultSourceAudioTrackSettings, currentTime, timelineTime: timelinePlayheadTime, duration, @@ -3751,6 +3772,7 @@ export default function VideoEditor() { if (id) { setSelectedZoomId(null); setSelectedAnnotationId(null); + setActiveEffectSection("audio"); } }, []); @@ -3768,6 +3790,7 @@ export default function VideoEditor() { setSelectedAudioId(id); setSelectedZoomId(null); setSelectedAnnotationId(null); + setActiveEffectSection("audio"); }, []); const handleAudioSpanChange = useCallback((id: string, span: Span, trackIndex?: number) => { diff --git a/src/components/video-editor/audio/useAudioPreviewSync.ts b/src/components/video-editor/audio/useAudioPreviewSync.ts index b5668194..00b2b3b9 100644 --- a/src/components/video-editor/audio/useAudioPreviewSync.ts +++ b/src/components/video-editor/audio/useAudioPreviewSync.ts @@ -274,13 +274,27 @@ export function useAudioPreviewSync({ : SOURCE_AUDIO_PREVIEW_PAUSED_SEEK_DRIFT_SECONDS; for (const audio of sourceAudioElementsRef.current.values()) { + const sourceAudioPath = audio.dataset.sourceAudioPath ?? ""; + audio.volume = isCurrentClipMuted + ? 0 + : Math.max(0, Math.min(1, previewVolume * getSourceTrackPreviewGain(sourceAudioPath))); + enablePitchPreservingPlayback(audio); const audioDuration = Number.isFinite(audio.duration) ? audio.duration : null; - const startDelaySeconds = estimateCompanionAudioStartDelaySeconds( + const isMicCompanionTrack = /\.mic\./i.test(sourceAudioPath); + const rawStartDelaySeconds = estimateCompanionAudioStartDelaySeconds( duration, audioDuration, - sourceAudioFallbackStartDelayMsByPath[audio.dataset.sourceAudioPath ?? ""], + sourceAudioFallbackStartDelayMsByPath[sourceAudioPath], ); + const maxPreviewStartDelaySeconds = isMicCompanionTrack ? 2 : 5; + const startDelaySeconds = isMicCompanionTrack + ? 0 + : Number.isFinite(duration) && + (rawStartDelaySeconds >= Math.max(0, duration - 0.01) || + rawStartDelaySeconds > Math.max(maxPreviewStartDelaySeconds, duration * 0.9)) + ? 0 + : rawStartDelaySeconds; const beforeAudioStart = currentTime + 0.001 < startDelaySeconds; const targetTime = clampMediaTimeToDuration(currentTime - startDelaySeconds, audioDuration); @@ -317,7 +331,10 @@ export function useAudioPreviewSync({ currentTime, duration, effectiveSpeedRegions, + getSourceTrackPreviewGain, + isCurrentClipMuted, isPlaying, + previewVolume, previewSourceAudioFallbackPaths, sourceAudioFallbackStartDelayMsByPath, ]); diff --git a/src/components/video-editor/audio/useClipAudioSettingsController.ts b/src/components/video-editor/audio/useClipAudioSettingsController.ts index 1873fb52..8c79ac09 100644 --- a/src/components/video-editor/audio/useClipAudioSettingsController.ts +++ b/src/components/video-editor/audio/useClipAudioSettingsController.ts @@ -1,18 +1,31 @@ -import { useCallback, useMemo } from "react"; +import React, { useCallback, useMemo } from "react"; import { SOURCE_AUDIO_NORMALIZE_GAIN, getSourceTrackIdFromPath, } from "./sourceAudioTracks"; import { useSourceAudioTrackSettings } from "./useSourceAudioTrackSettings"; +import { SourceAudioTrackSettings } from "../types"; interface UseClipAudioSettingsControllerParams { selectedClipId: string | null; activeClipId: string | null; + sourceAudioTrackSettingsByClip: Record; + setSourceAudioTrackSettingsByClip: React.Dispatch< + React.SetStateAction> + >; + defaultSourceAudioTrackSettings: SourceAudioTrackSettings; + setDefaultSourceAudioTrackSettings: React.Dispatch< + React.SetStateAction + >; } export function useClipAudioSettingsController({ selectedClipId, activeClipId, + sourceAudioTrackSettingsByClip, + setSourceAudioTrackSettingsByClip, + defaultSourceAudioTrackSettings, + setDefaultSourceAudioTrackSettings, }: UseClipAudioSettingsControllerParams) { const { sourceAudioTrackMeta, @@ -25,25 +38,45 @@ export function useClipAudioSettingsController({ } = useSourceAudioTrackSettings({ selectedClipId, activeClipId, + sourceAudioTrackSettingsByClip, + setSourceAudioTrackSettingsByClip, + defaultSourceAudioTrackSettings, + setDefaultSourceAudioTrackSettings, }); + const previewSourceAudioTrackSettings = useMemo( + () => + activeClipId ? activeSourceAudioTrackSettings : selectedClipSourceAudioTrackSettings, + [activeClipId, activeSourceAudioTrackSettings, selectedClipSourceAudioTrackSettings], + ); + + const embeddedTrackId = useMemo<"mixed" | "system">(() => { + const hasMixedTrack = sourceAudioTrackMeta.some((track) => track.id === "mixed"); + if (hasMixedTrack) return "mixed"; + const hasSystemTrack = sourceAudioTrackMeta.some((track) => track.id === "system"); + return hasSystemTrack ? "system" : "mixed"; + }, [sourceAudioTrackMeta]); + const embeddedSourcePreviewGain = useMemo(() => { - const settings = activeSourceAudioTrackSettings.mixed ?? { volume: 1, normalize: false }; + const settings = previewSourceAudioTrackSettings[embeddedTrackId] ?? { + volume: 1, + normalize: false, + }; const normalizeGain = settings.normalize ? SOURCE_AUDIO_NORMALIZE_GAIN : 1; return Math.max(0, Math.min(2, settings.volume * normalizeGain)); - }, [activeSourceAudioTrackSettings]); + }, [embeddedTrackId, previewSourceAudioTrackSettings]); const getSourceTrackPreviewGain = useCallback( (audioPath: string) => { const trackId = getSourceTrackIdFromPath(audioPath); - const settings = activeSourceAudioTrackSettings[trackId] ?? { + const settings = previewSourceAudioTrackSettings[trackId] ?? { volume: 1, normalize: false, }; const normalizeGain = settings.normalize ? SOURCE_AUDIO_NORMALIZE_GAIN : 1; return Math.max(0, Math.min(2, settings.volume * normalizeGain)); }, - [activeSourceAudioTrackSettings], + [previewSourceAudioTrackSettings], ); return { diff --git a/src/components/video-editor/audio/useSourceAudioTrackSettings.ts b/src/components/video-editor/audio/useSourceAudioTrackSettings.ts index 5a8ca053..34ed0ee3 100644 --- a/src/components/video-editor/audio/useSourceAudioTrackSettings.ts +++ b/src/components/video-editor/audio/useSourceAudioTrackSettings.ts @@ -1,12 +1,17 @@ -import { useCallback, useMemo, useState } from "react"; +import React, { useCallback, useMemo, useState } from "react"; +import type { SourceAudioTrackSettings } from "../types"; -export type SourceAudioTrackSetting = { volume: number; normalize: boolean }; -export type SourceAudioTrackSettings = Record; export type SourceAudioTrackMeta = Array<{ id: string; label: string }>; interface UseSourceAudioTrackSettingsParams { selectedClipId: string | null; activeClipId: string | null; + sourceAudioTrackSettingsByClip: Record; + setSourceAudioTrackSettingsByClip: React.Dispatch< + React.SetStateAction> + >; + defaultSourceAudioTrackSettings: SourceAudioTrackSettings; + setDefaultSourceAudioTrackSettings: React.Dispatch>; } export interface UseSourceAudioTrackSettingsResult { @@ -22,14 +27,12 @@ export interface UseSourceAudioTrackSettingsResult { export function useSourceAudioTrackSettings({ selectedClipId, activeClipId, + sourceAudioTrackSettingsByClip, + setSourceAudioTrackSettingsByClip, + defaultSourceAudioTrackSettings, + setDefaultSourceAudioTrackSettings, }: UseSourceAudioTrackSettingsParams): UseSourceAudioTrackSettingsResult { const [sourceAudioTrackMeta, setSourceAudioTrackMeta] = useState([]); - const [sourceAudioTrackSettingsByClip, setSourceAudioTrackSettingsByClip] = useState< - Record - >({}); - const [defaultSourceAudioTrackSettings, setDefaultSourceAudioTrackSettings] = useState< - SourceAudioTrackSettings - >({}); const activeSourceAudioTrackSettings = useMemo(() => { if (!activeClipId) { diff --git a/src/components/video-editor/audio/useVideoEditorAudio.ts b/src/components/video-editor/audio/useVideoEditorAudio.ts index 02de6a23..d31f7d24 100644 --- a/src/components/video-editor/audio/useVideoEditorAudio.ts +++ b/src/components/video-editor/audio/useVideoEditorAudio.ts @@ -1,6 +1,11 @@ -import { useMemo } from "react"; +import React, { useMemo } from "react"; import { resolveSourceAudioFallbackPaths } from "@/lib/exporter/sourceAudioFallback"; -import type { AudioRegion, ClipRegion, SpeedRegion } from "../types"; +import type { + AudioRegion, + ClipRegion, + SourceAudioTrackSettings, + SpeedRegion, +} from "../types"; import { getActiveClipIdAtSourceTime, isClipMutedById } from "./clipAudio"; import { useAudioPreviewSync } from "./useAudioPreviewSync"; import { useClipAudioSettingsController } from "./useClipAudioSettingsController"; @@ -27,6 +32,14 @@ interface UseVideoEditorAudioParams { clipRegions: ClipRegion[]; audioRegions: AudioRegion[]; effectiveSpeedRegions: SpeedRegion[]; + sourceAudioTrackSettingsByClip: Record; + setSourceAudioTrackSettingsByClip: React.Dispatch< + React.SetStateAction> + >; + defaultSourceAudioTrackSettings: SourceAudioTrackSettings; + setDefaultSourceAudioTrackSettings: React.Dispatch< + React.SetStateAction + >; currentTime: number; timelineTime: number; duration: number; @@ -42,6 +55,10 @@ export function useVideoEditorAudio({ clipRegions, audioRegions, effectiveSpeedRegions, + sourceAudioTrackSettingsByClip, + setSourceAudioTrackSettingsByClip, + defaultSourceAudioTrackSettings, + setDefaultSourceAudioTrackSettings, currentTime, timelineTime, duration, @@ -65,8 +82,12 @@ export function useVideoEditorAudio({ () => resolveSourceAudioFallbackPaths(currentSourcePath, sourceAudioFallbackPaths), [currentSourcePath, sourceAudioFallbackPaths], ); + const hasSystemCompanionPreviewTrack = previewSourceAudioFallbackPaths.some((audioPath) => + audioPath.toLowerCase().includes(".system."), + ); const shouldMutePreviewVideo = - !hasEmbeddedSourceAudio && previewSourceAudioFallbackPaths.length > 0; + previewSourceAudioFallbackPaths.length > 0 && + (!hasEmbeddedSourceAudio || hasSystemCompanionPreviewTrack); const activeClipIdAtCurrentTime = useMemo( () => getActiveClipIdAtSourceTime(currentTime, clipRegions), @@ -90,6 +111,10 @@ export function useVideoEditorAudio({ } = useClipAudioSettingsController({ selectedClipId, activeClipId: activeClipIdAtCurrentTime, + sourceAudioTrackSettingsByClip, + setSourceAudioTrackSettingsByClip, + defaultSourceAudioTrackSettings, + setDefaultSourceAudioTrackSettings, }); useAudioPreviewSync({ diff --git a/src/components/video-editor/projectPersistence.ts b/src/components/video-editor/projectPersistence.ts index a75deb78..49f74348 100644 --- a/src/components/video-editor/projectPersistence.ts +++ b/src/components/video-editor/projectPersistence.ts @@ -62,6 +62,7 @@ import { DEFAULT_ZOOM_SMOOTHNESS, getDefaultCaptionFontFamily, type Padding, + SourceAudioTrackSettings, type SpeedRegion, type TrimRegion, type WebcamOverlaySettings, @@ -127,6 +128,8 @@ export interface ProjectEditorState { autoCaptionSettings: AutoCaptionSettings; webcam: WebcamOverlaySettings; aspectRatio: AspectRatio; + sourceAudioTrackSettingsByClip?: Record; + defaultSourceAudioTrackSettings?: SourceAudioTrackSettings; exportEncodingMode: ExportEncodingMode; exportBackendPreference: ExportBackendPreference; exportPipelineModel: ExportPipelineModel; @@ -496,6 +499,10 @@ export function normalizeProjectEditor(editor: Partial): Pro endMs, speed: isFiniteNumber(region.speed) ? region.speed : 1, muted: typeof region.muted === "boolean" ? region.muted : false, + showSourceAudio: + typeof region.showSourceAudio === "boolean" + ? region.showSourceAudio + : false, }; }) : []; @@ -983,6 +990,16 @@ export function normalizeProjectEditor(editor: Partial): Pro ? clamp(webcam.margin, 0, 96) : DEFAULT_WEBCAM_MARGIN, }, + sourceAudioTrackSettingsByClip: + editor.sourceAudioTrackSettingsByClip && + typeof editor.sourceAudioTrackSettingsByClip === "object" + ? editor.sourceAudioTrackSettingsByClip + : {}, + defaultSourceAudioTrackSettings: + editor.defaultSourceAudioTrackSettings && + typeof editor.defaultSourceAudioTrackSettings === "object" + ? editor.defaultSourceAudioTrackSettings + : {}, aspectRatio: typeof editor.aspectRatio === "string" && (validAspectRatios.has(editor.aspectRatio as AspectRatio) || diff --git a/src/components/video-editor/timeline/TimelineEditor.tsx b/src/components/video-editor/timeline/TimelineEditor.tsx index 8fb8a540..28e83a5f 100644 --- a/src/components/video-editor/timeline/TimelineEditor.tsx +++ b/src/components/video-editor/timeline/TimelineEditor.tsx @@ -291,12 +291,30 @@ const TimelineEditor = forwardRef( const sourceAudioTracks = useMemo>(() => { if (systemSidecarPeaks || micSidecarPeaks) { const tracks: Array<{ id: string; label: string; peaks: AudioPeaksData }> = []; - if (systemSidecarPeaks) tracks.push({ id: "system", label: "Source System", peaks: systemSidecarPeaks }); - if (micSidecarPeaks) tracks.push({ id: "mic", label: "Source Mic", peaks: micSidecarPeaks }); + if (systemSidecarPeaks) + tracks.push({ + id: "system", + label: t("audio.systemLabel", "Source System"), + peaks: systemSidecarPeaks, + }); + if (micSidecarPeaks) + tracks.push({ + id: "mic", + label: t("audio.micLabel", "Source Mic"), + peaks: micSidecarPeaks, + }); return tracks; } - return sourceAudioPeaks ? [{ id: "mixed", label: "Source", peaks: sourceAudioPeaks }] : []; - }, [micSidecarPeaks, sourceAudioPeaks, systemSidecarPeaks]); + return sourceAudioPeaks + ? [ + { + id: "mixed", + label: t("audio.mixedLabel", "Source"), + peaks: sourceAudioPeaks, + }, + ] + : []; + }, [micSidecarPeaks, sourceAudioPeaks, systemSidecarPeaks, t]); useEffect(() => { onSourceAudioTracksMetaChange?.(sourceAudioTracks.map((t) => ({ id: t.id, label: t.label }))); }, [onSourceAudioTracksMetaChange, sourceAudioTracks]); diff --git a/src/components/video-editor/types.ts b/src/components/video-editor/types.ts index ec65c44e..838b4646 100644 --- a/src/components/video-editor/types.ts +++ b/src/components/video-editor/types.ts @@ -64,6 +64,7 @@ export type EditorEffectSection = | "crop" | "extensions" | "clip" + | "audio" | `ext:${string}`; export type ZoomTransitionEasing = "recordly" | "glide" | "smooth" | "snappy" | "linear"; @@ -466,6 +467,13 @@ export const DEFAULT_PADDING: Padding = { linked: true, }; +export interface SourceAudioTrackSetting { + volume: number; + normalize: boolean; +} + +export type SourceAudioTrackSettings = Record; + export interface AudioRegion { id: string; startMs: number; diff --git a/src/i18n/locales/en/settings.json b/src/i18n/locales/en/settings.json index 7b942498..e8ebfebf 100644 --- a/src/i18n/locales/en/settings.json +++ b/src/i18n/locales/en/settings.json @@ -206,6 +206,9 @@ }, "audio": { "sourceTracksTitle": "Clip Source Audio", + "systemLabel": "Source System", + "micLabel": "Source Mic", + "mixedLabel": "Source", "volumeTitle": "Volume", "normalize": "Normalize", "deleteRegion": "Delete Audio" diff --git a/src/i18n/locales/es/settings.json b/src/i18n/locales/es/settings.json index 75f78d75..bec77412 100644 --- a/src/i18n/locales/es/settings.json +++ b/src/i18n/locales/es/settings.json @@ -186,6 +186,9 @@ }, "audio": { "sourceTracksTitle": "Audio fuente del clip", + "systemLabel": "Sonido del sistema", + "micLabel": "Micrófono", + "mixedLabel": "Fuente", "volumeTitle": "Volumen", "normalize": "Normalizar", "deleteRegion": "Eliminar audio" diff --git a/src/i18n/locales/fr/settings.json b/src/i18n/locales/fr/settings.json index 241bc269..ce314c55 100644 --- a/src/i18n/locales/fr/settings.json +++ b/src/i18n/locales/fr/settings.json @@ -186,6 +186,9 @@ }, "audio": { "sourceTracksTitle": "Audio source du clip", + "systemLabel": "Son Système", + "micLabel": "Microphone", + "mixedLabel": "Source", "volumeTitle": "Volume", "normalize": "Normaliser", "deleteRegion": "Supprimer l'audio" diff --git a/src/i18n/locales/zh-CN/settings.json b/src/i18n/locales/zh-CN/settings.json index f895653f..02c3868a 100644 --- a/src/i18n/locales/zh-CN/settings.json +++ b/src/i18n/locales/zh-CN/settings.json @@ -197,6 +197,15 @@ "saveProject": "保存项目", "exportVideo": "导出{{format}}", "reportBug": "报告问题", - "starOnGithub": "在 GitHub 上加星" + "starOnGithub": "在 GitHub 上点赞" + }, + "audio": { + "sourceTracksTitle": "片段原始音频", + "systemLabel": "系统声音", + "micLabel": "麦克风", + "mixedLabel": "来源", + "volumeTitle": "音量", + "normalize": "标准化", + "deleteRegion": "删除音频" } } From d4fe5d72a1b5a4c945305510b86ae26372b7f796 Mon Sep 17 00:00:00 2001 From: Alan Trebugeais Date: Sat, 9 May 2026 18:51:44 +0200 Subject: [PATCH 10/25] audio encoding works with exporting. --- src/components/video-editor/VideoEditor.tsx | 3 +- .../video-editor/audio/useAudioPreviewSync.ts | 132 +++++++++++++++--- .../video-editor/audio/useVideoEditorAudio.ts | 14 +- src/lib/exporter/audioEncoder.ts | 77 +++++++--- src/lib/exporter/modernVideoExporter.ts | 4 + .../exporter/sourceTrackRoutingPolicy.test.ts | 41 ++++++ src/lib/exporter/sourceTrackRoutingPolicy.ts | 50 +++++++ src/lib/exporter/videoExporter.ts | 4 + 8 files changed, 274 insertions(+), 51 deletions(-) create mode 100644 src/lib/exporter/sourceTrackRoutingPolicy.test.ts create mode 100644 src/lib/exporter/sourceTrackRoutingPolicy.ts diff --git a/src/components/video-editor/VideoEditor.tsx b/src/components/video-editor/VideoEditor.tsx index fbc61a63..5ca4c864 100644 --- a/src/components/video-editor/VideoEditor.tsx +++ b/src/components/video-editor/VideoEditor.tsx @@ -4405,6 +4405,7 @@ export default function VideoEditor() { sourceAudioFallbackPaths: audio.sourceAudioFallbackPaths, sourceAudioFallbackStartDelayMsByPath: audio.sourceAudioFallbackStartDelayMsByPath, + sourceAudioTrackSettings: audio.activeSourceAudioTrackSettings, previewWidth, previewHeight, onProgress: (progress: ExportProgress) => { @@ -6064,7 +6065,7 @@ export default function VideoEditor() { } cursorSway={cursorSway} volume={ - audio.isCurrentClipMuted + audio.shouldMutePreviewVideo || audio.isCurrentClipMuted ? 0 : Math.max( 0, diff --git a/src/components/video-editor/audio/useAudioPreviewSync.ts b/src/components/video-editor/audio/useAudioPreviewSync.ts index 00b2b3b9..3d05c858 100644 --- a/src/components/video-editor/audio/useAudioPreviewSync.ts +++ b/src/components/video-editor/audio/useAudioPreviewSync.ts @@ -10,9 +10,6 @@ import type { AudioRegion, SpeedRegion } from "../types"; const SOURCE_AUDIO_PREVIEW_PLAYING_SEEK_DRIFT_SECONDS = 0.18; const SOURCE_AUDIO_PREVIEW_PAUSED_SEEK_DRIFT_SECONDS = 0.01; -const SOURCE_AUDIO_PREVIEW_RATE_TOLERANCE_SECONDS = 0.08; -const SOURCE_AUDIO_PREVIEW_RATE_CORRECTION_WINDOW_SECONDS = 8; -const SOURCE_AUDIO_PREVIEW_MAX_RATE_ADJUSTMENT = 0.015; interface UseAudioPreviewSyncParams { audioRegions: AudioRegion[]; @@ -47,10 +44,43 @@ export function useAudioPreviewSync({ const audioElementRevokersRef = useRef void>>(new Map()); const audioElementResourcesRef = useRef>(new Map()); const sourceAudioElementsRef = useRef>(new Map()); + const sourceAudioMediaNodesRef = useRef>(new Map()); + const sourceAudioGainNodesRef = useRef>(new Map()); const sourceAudioElementRevokersRef = useRef void>>(new Map()); const sourceAudioElementResourcesRef = useRef>(new Map()); + const sourceAudioContextRef = useRef(null); + const sourceAudioMasterGainRef = useRef(null); + const sourceAudioResumePromiseRef = useRef | null>(null); const lastSourceAudioSyncTimeRef = useRef(null); + const ensureSourceAudioContext = () => { + if (!sourceAudioContextRef.current) { + const context = new AudioContext({ latencyHint: "interactive" }); + const masterGain = context.createGain(); + masterGain.gain.value = 1; + masterGain.connect(context.destination); + sourceAudioContextRef.current = context; + sourceAudioMasterGainRef.current = masterGain; + } + return sourceAudioContextRef.current; + }; + + const ensureSourceAudioRunning = () => { + const context = ensureSourceAudioContext(); + if (context.state === "running") { + return Promise.resolve(); + } + if (!sourceAudioResumePromiseRef.current) { + sourceAudioResumePromiseRef.current = context + .resume() + .catch(() => undefined) + .finally(() => { + sourceAudioResumePromiseRef.current = null; + }); + } + return sourceAudioResumePromiseRef.current; + }; + useEffect(() => { let cancelled = false; const existing = audioElementsRef.current; @@ -117,6 +147,10 @@ export function useAudioPreviewSync({ if (!currentIds.has(id)) { audio.pause(); audio.src = ""; + sourceAudioMediaNodesRef.current.get(id)?.disconnect(); + sourceAudioMediaNodesRef.current.delete(id); + sourceAudioGainNodesRef.current.get(id)?.disconnect(); + sourceAudioGainNodesRef.current.delete(id); sourceAudioElementRevokersRef.current.get(id)?.(); sourceAudioElementRevokersRef.current.delete(id); sourceAudioElementResourcesRef.current.delete(id); @@ -129,10 +163,27 @@ export function useAudioPreviewSync({ if (!audio) { audio = new Audio(); audio.preload = "auto"; + audio.crossOrigin = "anonymous"; existing.set(audioPath, audio); } + audio.volume = 1; audio.dataset.sourceAudioPath = audioPath; + const context = ensureSourceAudioContext(); + const masterGain = sourceAudioMasterGainRef.current; + if (context && masterGain && !sourceAudioMediaNodesRef.current.has(audioPath)) { + try { + const mediaNode = context.createMediaElementSource(audio); + const trackGainNode = context.createGain(); + mediaNode.connect(trackGainNode); + trackGainNode.connect(masterGain); + sourceAudioMediaNodesRef.current.set(audioPath, mediaNode); + sourceAudioGainNodesRef.current.set(audioPath, trackGainNode); + } catch (error) { + onSourceFallbackLoadError(error); + } + } + if (sourceAudioElementResourcesRef.current.get(audioPath) !== audioPath) { audio.pause(); audio.src = ""; @@ -174,9 +225,16 @@ export function useAudioPreviewSync({ })(); } - audio.volume = isCurrentClipMuted + const trackGainNode = sourceAudioGainNodesRef.current.get(audioPath); + if (trackGainNode) { + trackGainNode.gain.value = Math.max(0, Math.min(2, getSourceTrackPreviewGain(audioPath))); + } + } + + if (sourceAudioMasterGainRef.current) { + sourceAudioMasterGainRef.current.gain.value = isCurrentClipMuted ? 0 - : Math.max(0, Math.min(1, previewVolume * getSourceTrackPreviewGain(audioPath))); + : Math.max(0, Math.min(1, previewVolume)); } if (previewSourceAudioFallbackPaths.length === 0) { @@ -210,12 +268,30 @@ export function useAudioPreviewSync({ audio.pause(); audio.src = ""; } + for (const node of sourceAudioMediaNodesRef.current.values()) { + node.disconnect(); + } + for (const node of sourceAudioGainNodesRef.current.values()) { + node.disconnect(); + } for (const revoke of sourceAudioElementRevokersRef.current.values()) { revoke(); } sourceAudioElementsRef.current.clear(); + sourceAudioMediaNodesRef.current.clear(); + sourceAudioGainNodesRef.current.clear(); sourceAudioElementRevokersRef.current.clear(); sourceAudioElementResourcesRef.current.clear(); + if (sourceAudioMasterGainRef.current) { + sourceAudioMasterGainRef.current.disconnect(); + sourceAudioMasterGainRef.current = null; + } + const context = sourceAudioContextRef.current; + sourceAudioContextRef.current = null; + sourceAudioResumePromiseRef.current = null; + if (context) { + void context.close(); + } lastSourceAudioSyncTimeRef.current = null; }; }, []); @@ -272,12 +348,18 @@ export function useAudioPreviewSync({ const driftThreshold = isPlaying ? SOURCE_AUDIO_PREVIEW_PLAYING_SEEK_DRIFT_SECONDS : SOURCE_AUDIO_PREVIEW_PAUSED_SEEK_DRIFT_SECONDS; + if (sourceAudioMasterGainRef.current) { + sourceAudioMasterGainRef.current.gain.value = isCurrentClipMuted + ? 0 + : Math.max(0, Math.min(1, previewVolume)); + } for (const audio of sourceAudioElementsRef.current.values()) { const sourceAudioPath = audio.dataset.sourceAudioPath ?? ""; - audio.volume = isCurrentClipMuted - ? 0 - : Math.max(0, Math.min(1, previewVolume * getSourceTrackPreviewGain(sourceAudioPath))); + const trackGainNode = sourceAudioGainNodesRef.current.get(sourceAudioPath); + if (trackGainNode) { + trackGainNode.gain.value = Math.max(0, Math.min(2, getSourceTrackPreviewGain(sourceAudioPath))); + } enablePitchPreservingPlayback(audio); const audioDuration = Number.isFinite(audio.duration) ? audio.duration : null; @@ -298,7 +380,11 @@ export function useAudioPreviewSync({ const beforeAudioStart = currentTime + 0.001 < startDelaySeconds; const targetTime = clampMediaTimeToDuration(currentTime - startDelaySeconds, audioDuration); - if (timelineJumped || Math.abs(audio.currentTime - targetTime) > driftThreshold) { + const shouldSeek = + timelineJumped || + (!isPlaying && Math.abs(audio.currentTime - targetTime) > driftThreshold) || + (isPlaying && Math.abs(audio.currentTime - targetTime) > 0.9); + if (shouldSeek) { try { audio.currentTime = targetTime; } catch { @@ -306,21 +392,18 @@ export function useAudioPreviewSync({ } } - const syncedPlaybackRate = getMediaSyncPlaybackRate({ - basePlaybackRate: targetPlaybackRate, - currentTime: audio.currentTime, - targetTime, - toleranceSeconds: SOURCE_AUDIO_PREVIEW_RATE_TOLERANCE_SECONDS, - correctionWindowSeconds: SOURCE_AUDIO_PREVIEW_RATE_CORRECTION_WINDOW_SECONDS, - maxAdjustment: SOURCE_AUDIO_PREVIEW_MAX_RATE_ADJUSTMENT, - }); + // KISS for companion source tracks: fixed playback rate avoids audible flutter/stutter + // from continuous micro-corrections on system audio. + const syncedPlaybackRate = targetPlaybackRate; if (Math.abs(audio.playbackRate - syncedPlaybackRate) > 0.001) { audio.playbackRate = syncedPlaybackRate; } const atEnd = audioDuration !== null && targetTime >= audioDuration; if (isPlaying && !beforeAudioStart && !atEnd) { - audio.play().catch(() => undefined); + void ensureSourceAudioRunning().then(() => { + audio.play().catch(() => undefined); + }); } else if (!audio.paused) { audio.pause(); } @@ -338,4 +421,17 @@ export function useAudioPreviewSync({ previewSourceAudioFallbackPaths, sourceAudioFallbackStartDelayMsByPath, ]); + + useEffect(() => { + if (!isPlaying || previewSourceAudioFallbackPaths.length === 0) { + return; + } + void ensureSourceAudioRunning().then(() => { + for (const audio of sourceAudioElementsRef.current.values()) { + if (audio.paused) { + audio.play().catch(() => undefined); + } + } + }); + }, [isPlaying, previewSourceAudioFallbackPaths]); } diff --git a/src/components/video-editor/audio/useVideoEditorAudio.ts b/src/components/video-editor/audio/useVideoEditorAudio.ts index d31f7d24..d992e871 100644 --- a/src/components/video-editor/audio/useVideoEditorAudio.ts +++ b/src/components/video-editor/audio/useVideoEditorAudio.ts @@ -1,5 +1,5 @@ import React, { useMemo } from "react"; -import { resolveSourceAudioFallbackPaths } from "@/lib/exporter/sourceAudioFallback"; +import { resolveSourceTrackRoutingPolicy } from "@/lib/exporter/sourceTrackRoutingPolicy"; import type { AudioRegion, ClipRegion, @@ -78,16 +78,12 @@ export function useVideoEditorAudio({ summarizeErrorMessage, }); - const { hasEmbeddedSourceAudio, externalAudioPaths: previewSourceAudioFallbackPaths } = useMemo( - () => resolveSourceAudioFallbackPaths(currentSourcePath, sourceAudioFallbackPaths), + const sourceTrackRoutingPolicy = useMemo( + () => resolveSourceTrackRoutingPolicy(currentSourcePath, sourceAudioFallbackPaths), [currentSourcePath, sourceAudioFallbackPaths], ); - const hasSystemCompanionPreviewTrack = previewSourceAudioFallbackPaths.some((audioPath) => - audioPath.toLowerCase().includes(".system."), - ); - const shouldMutePreviewVideo = - previewSourceAudioFallbackPaths.length > 0 && - (!hasEmbeddedSourceAudio || hasSystemCompanionPreviewTrack); + const previewSourceAudioFallbackPaths = sourceTrackRoutingPolicy.playbackPaths; + const shouldMutePreviewVideo = sourceTrackRoutingPolicy.muteEmbeddedPreview; const activeClipIdAtCurrentTime = useMemo( () => getActiveClipIdAtSourceTime(currentTime, clipRegions), diff --git a/src/lib/exporter/audioEncoder.ts b/src/lib/exporter/audioEncoder.ts index 647c9768..c6f014f8 100644 --- a/src/lib/exporter/audioEncoder.ts +++ b/src/lib/exporter/audioEncoder.ts @@ -3,12 +3,16 @@ import type { AudioRegion, ClipRegion, SpeedRegion, + SourceAudioTrackSettings, TrimRegion, } from "@/components/video-editor/types"; import { estimateCompanionAudioStartDelaySeconds } from "@/lib/mediaTiming"; import { resolveMediaElementSource } from "./localMediaSource"; import type { VideoMuxer } from "./muxer"; -import { resolveSourceAudioFallbackPaths } from "./sourceAudioFallback"; +import { + getSourceTrackIdFromPath, + resolveSourceTrackRoutingPolicy, +} from "./sourceTrackRoutingPolicy"; const AUDIO_BITRATE = 128_000; const DECODE_BACKPRESSURE_LIMIT = 20; @@ -26,8 +30,8 @@ interface TimelineSlice { } interface PreparedOfflineRender { - mainBuffer: AudioBuffer | null; - companionEntries: Array<{ buffer: AudioBuffer; startDelaySec: number }>; + mainBufferEntry: { buffer: AudioBuffer; gain: number } | null; + companionEntries: Array<{ buffer: AudioBuffer; startDelaySec: number; gain: number }>; regionEntries: Array<{ buffer: AudioBuffer; region: AudioRegion }>; slices: TimelineSlice[]; outputDurationMs: number; @@ -144,6 +148,7 @@ export class AudioProcessor { audioRegions?: AudioRegion[], sourceAudioFallbackPaths?: string[], sourceAudioFallbackStartDelayMsByPath?: Record, + sourceAudioTrackSettings?: SourceAudioTrackSettings, ): Promise { const sortedTrims = trimRegions ? [...trimRegions].sort((a, b) => a.startMs - b.startMs) @@ -161,16 +166,16 @@ export class AudioProcessor { (audioPath) => typeof audioPath === "string" && audioPath.trim().length > 0, ) : []; - const { hasEmbeddedSourceAudio, externalAudioPaths } = resolveSourceAudioFallbackPaths( + const routingPolicy = resolveSourceTrackRoutingPolicy( videoUrl, sortedSourceAudioFallbackPaths, ); - const hasTimedCompanionAudio = externalAudioPaths.some( + const hasTimedCompanionAudio = routingPolicy.playbackPaths.some( (audioPath) => (sourceAudioFallbackStartDelayMsByPath?.[audioPath] ?? 0) > 0, ); const needsSourceAudioMixing = - externalAudioPaths.length > 1 || - (hasEmbeddedSourceAudio && externalAudioPaths.length > 0) || + routingPolicy.playbackPaths.length > 1 || + (routingPolicy.hasEmbeddedSourceAudio && routingPolicy.playbackPaths.length > 0) || hasTimedCompanionAudio; // When speed edits, audio regions, or multiple audio sources need mixing, use offline AudioContext pipeline. @@ -186,14 +191,15 @@ export class AudioProcessor { sortedAudioRegions, sortedSourceAudioFallbackPaths, sourceAudioFallbackStartDelayMsByPath, + sourceAudioTrackSettings, muxer, ); return; } // Single sidecar audio with no speed/audio edits: demux directly (skips slow real-time rendering). - if (!hasEmbeddedSourceAudio && externalAudioPaths.length === 1) { - const sidecarDemuxer = await this.loadAudioFileDemuxer(externalAudioPaths[0]); + if (!routingPolicy.hasEmbeddedSourceAudio && routingPolicy.playbackPaths.length === 1) { + const sidecarDemuxer = await this.loadAudioFileDemuxer(routingPolicy.playbackPaths[0]); if (sidecarDemuxer) { try { await this.processTrimOnlyAudio(sidecarDemuxer, muxer, sortedTrims); @@ -215,8 +221,9 @@ export class AudioProcessor { sortedTrims, [], [], - externalAudioPaths, + routingPolicy.playbackPaths, sourceAudioFallbackStartDelayMsByPath, + sourceAudioTrackSettings, muxer, ); return; @@ -263,6 +270,7 @@ export class AudioProcessor { audioRegions?: AudioRegion[], sourceAudioFallbackPaths?: string[], sourceAudioFallbackStartDelayMsByPath?: Record, + sourceAudioTrackSettings?: SourceAudioTrackSettings, ): Promise { const sortedTrims = trimRegions ? [...trimRegions].sort((a, b) => a.startMs - b.startMs) @@ -288,6 +296,7 @@ export class AudioProcessor { sortedAudioRegions, sortedSourceAudioFallbackPaths, sourceAudioFallbackStartDelayMsByPath, + sourceAudioTrackSettings, ); return this.renderToWavBlobChunked(prepared); } @@ -563,6 +572,7 @@ export class AudioProcessor { audioRegions: AudioRegion[], sourceAudioFallbackPaths: string[], sourceAudioFallbackStartDelayMsByPath: Record | undefined, + sourceAudioTrackSettings: SourceAudioTrackSettings | undefined, muxer: VideoMuxer, ): Promise { const prepared = await this.prepareOfflineRender( @@ -572,6 +582,7 @@ export class AudioProcessor { audioRegions, sourceAudioFallbackPaths, sourceAudioFallbackStartDelayMsByPath, + sourceAudioTrackSettings, ); if (this.cancelled) return; await this.renderAndEncodeChunked(prepared, muxer); @@ -584,31 +595,45 @@ export class AudioProcessor { audioRegions: AudioRegion[], sourceAudioFallbackPaths: string[], sourceAudioFallbackStartDelayMsByPath?: Record, + sourceAudioTrackSettings?: SourceAudioTrackSettings, ): Promise { if (this.cancelled) throw new Error("Export cancelled"); this.onProgress?.(0); - const { externalAudioPaths } = resolveSourceAudioFallbackPaths( + const routingPolicy = resolveSourceTrackRoutingPolicy( videoUrl, sourceAudioFallbackPaths, ); // Decode embedded source audio separately from companion sidecars. - const mainBuffer = await this.decodeAudioFromUrl(videoUrl); + const mainBuffer = routingPolicy.includeEmbeddedInExport + ? await this.decodeAudioFromUrl(videoUrl) + : null; + const mainBufferGainSettings = + sourceAudioTrackSettings?.mixed ?? sourceAudioTrackSettings?.system ?? null; + const mainBufferGain = mainBufferGainSettings + ? Math.max(0, Math.min(2, mainBufferGainSettings.volume)) + : 1; + const mainBufferEntry = mainBuffer ? { buffer: mainBuffer, gain: mainBufferGain } : null; if (this.cancelled) throw new Error("Export cancelled"); // Decode companion / sidecar audio files - const companionEntries: Array<{ buffer: AudioBuffer; startDelaySec: number }> = []; + const companionEntries: Array<{ buffer: AudioBuffer; startDelaySec: number; gain: number }> = + []; const refDuration = mainBuffer?.duration ?? - (externalAudioPaths.length > 0 ? await this.getMediaDurationSec(videoUrl) : 0); - for (const audioPath of externalAudioPaths) { + (routingPolicy.playbackPaths.length > 0 ? await this.getMediaDurationSec(videoUrl) : 0); + for (const audioPath of routingPolicy.playbackPaths) { if (this.cancelled) throw new Error("Export cancelled"); const buffer = await this.decodeAudioFromUrl(audioPath); if (!buffer) continue; companionEntries.push({ buffer, + gain: Math.max( + 0, + Math.min(2, sourceAudioTrackSettings?.[getSourceTrackIdFromPath(audioPath)]?.volume ?? 1), + ), startDelaySec: estimateCompanionAudioStartDelaySeconds( refDuration, buffer.duration, @@ -629,15 +654,15 @@ export class AudioProcessor { this.onProgress?.(0.2); // Determine source duration for timeline calculation - const primaryBuffer = mainBuffer ?? companionEntries[0]?.buffer ?? null; + const primaryBuffer = mainBufferEntry?.buffer ?? companionEntries[0]?.buffer ?? null; if (!primaryBuffer && regionEntries.length === 0) { throw new Error("No decodable audio sources found"); } let sourceDurationSec: number; - if (mainBuffer) { - sourceDurationSec = mainBuffer.duration; - } else if (externalAudioPaths.length > 0 || regionEntries.length > 0) { + if (mainBufferEntry?.buffer) { + sourceDurationSec = mainBufferEntry.buffer.duration; + } else if (routingPolicy.playbackPaths.length > 0 || regionEntries.length > 0) { sourceDurationSec = await this.getMediaDurationSec(videoUrl); } else { sourceDurationSec = primaryBuffer?.duration ?? 0; @@ -661,7 +686,7 @@ export class AudioProcessor { const numChannels = Math.min(primaryBuffer?.numberOfChannels ?? 2, 2); return { - mainBuffer, + mainBufferEntry, companionEntries, regionEntries, slices, @@ -788,12 +813,13 @@ export class AudioProcessor { ); // Schedule main audio - if (prepared.mainBuffer) { + if (prepared.mainBufferEntry) { this.scheduleBufferThroughTimeline( offlineCtx, - prepared.mainBuffer, + prepared.mainBufferEntry.buffer, slices, 0, + prepared.mainBufferEntry.gain, outputOffsetSec, chunkSec, ); @@ -806,6 +832,7 @@ export class AudioProcessor { entry.buffer, slices, entry.startDelaySec, + entry.gain, outputOffsetSec, chunkSec, ); @@ -1265,6 +1292,7 @@ export class AudioProcessor { buffer: AudioBuffer, slices: TimelineSlice[], sourceStartDelaySec: number, + gain = 1, chunkOutputStartSec = 0, chunkDurationSec = Number.POSITIVE_INFINITY, ): void { @@ -1331,9 +1359,12 @@ export class AudioProcessor { } const source = ctx.createBufferSource(); + const gainNode = ctx.createGain(); + gainNode.gain.value = Math.max(0, Math.min(2, gain)); source.buffer = buffer; source.playbackRate.value = slice.speed; - source.connect(ctx.destination); + source.connect(gainNode); + gainNode.connect(ctx.destination); source.start(localOutputStartSec, effectiveBufferStartSec, effectiveSourceDurationSec); diff --git a/src/lib/exporter/modernVideoExporter.ts b/src/lib/exporter/modernVideoExporter.ts index ce8b3eae..424df4a0 100644 --- a/src/lib/exporter/modernVideoExporter.ts +++ b/src/lib/exporter/modernVideoExporter.ts @@ -8,6 +8,7 @@ import type { CursorTelemetryPoint, Padding, SpeedRegion, + SourceAudioTrackSettings, TrimRegion, WebcamOverlaySettings, ZoomMotionBlurTuning, @@ -137,6 +138,7 @@ interface VideoExporterConfig extends ExportConfig { audioRegions?: AudioRegion[]; sourceAudioFallbackPaths?: string[]; sourceAudioFallbackStartDelayMsByPath?: Record; + sourceAudioTrackSettings?: SourceAudioTrackSettings; previewWidth?: number; previewHeight?: number; onProgress?: (progress: ExportProgress) => void; @@ -752,6 +754,7 @@ export class ModernVideoExporter { this.config.audioRegions, this.config.sourceAudioFallbackPaths, this.config.sourceAudioFallbackStartDelayMsByPath, + this.config.sourceAudioTrackSettings, ), "audio processing", "audio", @@ -1805,6 +1808,7 @@ export class ModernVideoExporter { this.config.audioRegions, this.config.sourceAudioFallbackPaths, this.config.sourceAudioFallbackStartDelayMsByPath, + this.config.sourceAudioTrackSettings, ), description, "audio", diff --git a/src/lib/exporter/sourceTrackRoutingPolicy.test.ts b/src/lib/exporter/sourceTrackRoutingPolicy.test.ts new file mode 100644 index 00000000..ca16cd9b --- /dev/null +++ b/src/lib/exporter/sourceTrackRoutingPolicy.test.ts @@ -0,0 +1,41 @@ +import { describe, expect, it } from "vitest"; +import { resolveSourceTrackRoutingPolicy } from "./sourceTrackRoutingPolicy"; + +describe("resolveSourceTrackRoutingPolicy", () => { + it("prioritizes system+mic sidecars and mutes embedded preview", () => { + const policy = resolveSourceTrackRoutingPolicy("/tmp/recording.mp4", [ + "/tmp/recording.mp4", + "/tmp/recording.system.wav", + "/tmp/recording.mic.wav", + "/tmp/recording.mixed.wav", + ]); + + expect(policy.playbackPaths).toEqual([ + "/tmp/recording.system.wav", + "/tmp/recording.mic.wav", + ]); + expect(policy.muteEmbeddedPreview).toBe(true); + expect(policy.includeEmbeddedInExport).toBe(false); + }); + + it("falls back to mixed when dedicated tracks are absent", () => { + const policy = resolveSourceTrackRoutingPolicy("/tmp/recording.mp4", [ + "/tmp/recording.mixed.wav", + ]); + + expect(policy.playbackPaths).toEqual(["/tmp/recording.mixed.wav"]); + expect(policy.muteEmbeddedPreview).toBe(false); + expect(policy.includeEmbeddedInExport).toBe(false); + }); + + it("keeps embedded audio when only mic sidecar is present", () => { + const policy = resolveSourceTrackRoutingPolicy("/tmp/recording.mp4", [ + "/tmp/recording.mp4", + "/tmp/recording.mic.wav", + ]); + + expect(policy.playbackPaths).toEqual(["/tmp/recording.mic.wav"]); + expect(policy.muteEmbeddedPreview).toBe(true); + expect(policy.includeEmbeddedInExport).toBe(true); + }); +}); diff --git a/src/lib/exporter/sourceTrackRoutingPolicy.ts b/src/lib/exporter/sourceTrackRoutingPolicy.ts new file mode 100644 index 00000000..67601362 --- /dev/null +++ b/src/lib/exporter/sourceTrackRoutingPolicy.ts @@ -0,0 +1,50 @@ +import { resolveSourceAudioFallbackPaths } from "./sourceAudioFallback"; + +export type SourceTrackId = "mic" | "system" | "mixed"; + +export function getSourceTrackIdFromPath(audioPath: string): SourceTrackId { + const normalized = audioPath.toLowerCase(); + if (normalized.includes(".mic.")) return "mic"; + if (normalized.includes(".system.")) return "system"; + return "mixed"; +} + +export interface SourceTrackRoutingPolicy { + hasEmbeddedSourceAudio: boolean; + pathsByTrack: Partial>; + playbackPaths: string[]; + muteEmbeddedPreview: boolean; + includeEmbeddedInExport: boolean; +} + +export function resolveSourceTrackRoutingPolicy( + videoResource: string | null | undefined, + sourceAudioFallbackPaths: string[] | null | undefined, +): SourceTrackRoutingPolicy { + const { hasEmbeddedSourceAudio, externalAudioPaths } = resolveSourceAudioFallbackPaths( + videoResource, + sourceAudioFallbackPaths, + ); + + const pathsByTrack: Partial> = {}; + for (const path of externalAudioPaths) { + const trackId = getSourceTrackIdFromPath(path); + if (!pathsByTrack[trackId]) { + pathsByTrack[trackId] = path; + } + } + + const hasDedicatedTracks = Boolean(pathsByTrack.system || pathsByTrack.mic); + const playbackPaths: string[] = []; + if (pathsByTrack.system) playbackPaths.push(pathsByTrack.system); + if (pathsByTrack.mic) playbackPaths.push(pathsByTrack.mic); + if (!hasDedicatedTracks && pathsByTrack.mixed) playbackPaths.push(pathsByTrack.mixed); + + return { + hasEmbeddedSourceAudio, + pathsByTrack, + playbackPaths, + muteEmbeddedPreview: hasDedicatedTracks, + includeEmbeddedInExport: !pathsByTrack.system && !pathsByTrack.mixed, + }; +} diff --git a/src/lib/exporter/videoExporter.ts b/src/lib/exporter/videoExporter.ts index cf8f6337..55b1b184 100644 --- a/src/lib/exporter/videoExporter.ts +++ b/src/lib/exporter/videoExporter.ts @@ -8,6 +8,7 @@ import type { CursorTelemetryPoint, Padding, SpeedRegion, + SourceAudioTrackSettings, TrimRegion, WebcamOverlaySettings, ZoomMotionBlurTuning, @@ -92,6 +93,7 @@ interface VideoExporterConfig extends ExportConfig { audioRegions?: AudioRegion[]; sourceAudioFallbackPaths?: string[]; sourceAudioFallbackStartDelayMsByPath?: Record; + sourceAudioTrackSettings?: SourceAudioTrackSettings; previewWidth?: number; previewHeight?: number; onProgress?: (progress: ExportProgress) => void; @@ -398,6 +400,7 @@ export class VideoExporter { this.config.audioRegions, this.config.sourceAudioFallbackPaths, this.config.sourceAudioFallbackStartDelayMsByPath, + this.config.sourceAudioTrackSettings, ), "audio processing", "audio", @@ -847,6 +850,7 @@ export class VideoExporter { this.config.audioRegions, this.config.sourceAudioFallbackPaths, this.config.sourceAudioFallbackStartDelayMsByPath, + this.config.sourceAudioTrackSettings, ), "native edited audio rendering", "audio", From 4d5a7ceb6ddb60f7d8d0afd68a2fa43c3e398fd3 Mon Sep 17 00:00:00 2001 From: Alan Trebugeais Date: Sat, 9 May 2026 19:14:28 +0200 Subject: [PATCH 11/25] fix all audios issues it seems finally. with WebAudio API instead of html audio. --- src/components/video-editor/SettingsPanel.tsx | 30 ++-- src/components/video-editor/VideoEditor.tsx | 66 ++++++--- .../video-editor/audio/sourceAudioTracks.ts | 10 +- .../video-editor/audio/useAudioPreviewSync.ts | 78 +++++++---- .../audio/useSourceAudioTrackSettings.ts | 35 ++++- .../video-editor/projectPersistence.ts | 21 +-- .../components/viewport/TimelineCanvas.tsx | 22 +-- .../timeline/core/timelineTypes.ts | 2 + .../timeline/model/timelineModel.ts | 2 + src/components/video-editor/types.ts | 1 + src/lib/exporter/audioEncoder.ts | 42 ++++-- src/lib/exporter/audioRoutingEngine.ts | 129 ++++++++++++++++++ src/lib/exporter/sourceTrackRoutingPolicy.ts | 43 ++---- 13 files changed, 352 insertions(+), 129 deletions(-) create mode 100644 src/lib/exporter/audioRoutingEngine.ts diff --git a/src/components/video-editor/SettingsPanel.tsx b/src/components/video-editor/SettingsPanel.tsx index 88dd2659..53d42c7b 100644 --- a/src/components/video-editor/SettingsPanel.tsx +++ b/src/components/video-editor/SettingsPanel.tsx @@ -481,7 +481,9 @@ interface SettingsPanelProps { onClipDelete?: (id: string) => void; selectedAudioId?: string | null; selectedAudioVolume?: number | null; + selectedAudioNormalize?: boolean | null; onAudioVolumeChange?: (volume: number) => void; + onAudioNormalizeChange?: (normalize: boolean) => void; onAudioDelete?: (id: string) => void; shadowIntensity?: number; onShadowChange?: (intensity: number) => void; @@ -881,7 +883,9 @@ export function SettingsPanel({ onClipDelete, selectedAudioId, selectedAudioVolume, + selectedAudioNormalize, onAudioVolumeChange, + onAudioNormalizeChange, onAudioDelete, shadowIntensity = 0.67, onShadowChange, @@ -3049,16 +3053,16 @@ export function SettingsPanel({ ); - const audioSectionContent = ( -
+ const audioSectionContent = ( +
{tSettings("audio.volumeTitle", "Audio")} {Math.round((selectedAudioVolume ?? 1) * 100)}%
- onAudioVolumeChange?.(v)} formatValue={(v) => `${Math.round(v * 100)}%`} - parseInput={(text) => parseFloat(text.replace(/%$/, "")) / 100} - /> -
- ); + parseInput={(text) => parseFloat(text.replace(/%$/, "")) / 100} + /> +
+ + {tSettings("audio.normalize", "Normalize")} + + onAudioNormalizeChange?.(v)} + className="data-[state=checked]:bg-[#2563EB] scale-75" + /> +
+
+ ); const clipSectionContent = (
diff --git a/src/components/video-editor/VideoEditor.tsx b/src/components/video-editor/VideoEditor.tsx index 5ca4c864..83992392 100644 --- a/src/components/video-editor/VideoEditor.tsx +++ b/src/components/video-editor/VideoEditor.tsx @@ -3776,16 +3776,17 @@ export default function VideoEditor() { } }, []); - const handleAudioAdded = useCallback((span: Span, audioPath: string, trackIndex?: number) => { - const id = `audio-${nextAudioIdRef.current++}`; - const newRegion: AudioRegion = { - id, - startMs: Math.round(span.start), - endMs: Math.round(span.end), - audioPath, - volume: 1, - trackIndex, - }; + const handleAudioAdded = useCallback((span: Span, audioPath: string, trackIndex?: number) => { + const id = `audio-${nextAudioIdRef.current++}`; + const newRegion: AudioRegion = { + id, + startMs: Math.round(span.start), + endMs: Math.round(span.end), + audioPath, + volume: 1, + normalize: false, + trackIndex, + }; setAudioRegions((prev) => [...prev, newRegion]); setSelectedAudioId(id); setSelectedZoomId(null); @@ -3835,15 +3836,29 @@ export default function VideoEditor() { [selectedAudioId], ); - const handleAudioDelete = useCallback( - (id: string) => { + const handleAudioDelete = useCallback( + (id: string) => { setAudioRegions((prev) => prev.filter((region) => region.id !== id)); if (selectedAudioId === id) { setSelectedAudioId(null); } }, - [selectedAudioId], - ); + [selectedAudioId], + ); + + const handleAudioNormalizeChange = useCallback( + (normalize: boolean) => { + if (!selectedAudioId) { + return; + } + setAudioRegions((prev) => + prev.map((region) => + region.id === selectedAudioId ? { ...region, normalize } : region, + ), + ); + }, + [selectedAudioId], + ); const handleAnnotationAdded = useCallback((span: Span, trackIndex = 0) => { const id = `annotation-${nextAnnotationIdRef.current++}`; @@ -5761,14 +5776,21 @@ export default function VideoEditor() { audio.onSelectedClipSourceAudioTrackNormalizeChange } selectedAudioId={selectedAudioId} - selectedAudioVolume={ - selectedAudioId - ? (audioRegions.find((r) => r.id === selectedAudioId) - ?.volume ?? null) - : null - } - onAudioVolumeChange={handleAudioVolumeChange} - onAudioDelete={handleAudioDelete} + selectedAudioVolume={ + selectedAudioId + ? (audioRegions.find((r) => r.id === selectedAudioId) + ?.volume ?? null) + : null + } + selectedAudioNormalize={ + selectedAudioId + ? (audioRegions.find((r) => r.id === selectedAudioId) + ?.normalize ?? false) + : null + } + onAudioVolumeChange={handleAudioVolumeChange} + onAudioNormalizeChange={handleAudioNormalizeChange} + onAudioDelete={handleAudioDelete} shadowIntensity={shadowIntensity} onShadowChange={setShadowIntensity} backgroundBlur={backgroundBlur} diff --git a/src/components/video-editor/audio/sourceAudioTracks.ts b/src/components/video-editor/audio/sourceAudioTracks.ts index 8b9998c3..b3ea2cee 100644 --- a/src/components/video-editor/audio/sourceAudioTracks.ts +++ b/src/components/video-editor/audio/sourceAudioTracks.ts @@ -1,9 +1,5 @@ +import { getSourceTrackIdFromPath } from "@/lib/exporter/sourceTrackRoutingPolicy"; + export const SOURCE_AUDIO_FALLBACK_TOAST_ID = "source-audio-fallback-error"; export const SOURCE_AUDIO_NORMALIZE_GAIN = 1.35; - -export function getSourceTrackIdFromPath(audioPath: string): "mic" | "system" | "mixed" { - const normalized = audioPath.toLowerCase(); - if (normalized.includes(".mic.")) return "mic"; - if (normalized.includes(".system.")) return "system"; - return "mixed"; -} +export { getSourceTrackIdFromPath }; diff --git a/src/components/video-editor/audio/useAudioPreviewSync.ts b/src/components/video-editor/audio/useAudioPreviewSync.ts index 3d05c858..78482460 100644 --- a/src/components/video-editor/audio/useAudioPreviewSync.ts +++ b/src/components/video-editor/audio/useAudioPreviewSync.ts @@ -1,4 +1,5 @@ -import { useEffect, useRef } from "react"; +import { useEffect, useMemo, useRef } from "react"; +import { buildResolvedAudioPlan } from "@/lib/exporter/audioRoutingEngine"; import { resolveMediaElementSource } from "@/lib/exporter/localMediaSource"; import { clampMediaTimeToDuration, @@ -40,6 +41,24 @@ export function useAudioPreviewSync({ getSourceTrackPreviewGain, onSourceFallbackLoadError, }: UseAudioPreviewSyncParams) { + const resolvedPlan = useMemo( + () => + buildResolvedAudioPlan({ + videoResource: null, + sourceAudioFallbackPaths: previewSourceAudioFallbackPaths, + audioRegions, + }), + [audioRegions, previewSourceAudioFallbackPaths], + ); + const resolvedUserTracks = useMemo( + () => resolvedPlan.tracks.filter((track) => track.kind === "user"), + [resolvedPlan], + ); + const resolvedSourceTracks = useMemo( + () => resolvedPlan.tracks.filter((track) => track.kind !== "user"), + [resolvedPlan], + ); + const audioElementsRef = useRef>(new Map()); const audioElementRevokersRef = useRef void>>(new Map()); const audioElementResourcesRef = useRef>(new Map()); @@ -84,7 +103,7 @@ export function useAudioPreviewSync({ useEffect(() => { let cancelled = false; const existing = audioElementsRef.current; - const currentIds = new Set(audioRegions.map((r) => r.id)); + const currentIds = new Set(resolvedUserTracks.map((track) => track.id)); for (const [id, audio] of existing) { if (!currentIds.has(id)) { @@ -97,51 +116,51 @@ export function useAudioPreviewSync({ } } - for (const region of audioRegions) { - let audio = existing.get(region.id); + for (const track of resolvedUserTracks) { + let audio = existing.get(track.id); if (!audio) { audio = new Audio(); audio.preload = "auto"; - existing.set(region.id, audio); + existing.set(track.id, audio); } - if (audioElementResourcesRef.current.get(region.id) !== region.audioPath) { + if (audioElementResourcesRef.current.get(track.id) !== track.sourceRef.path) { audio.pause(); audio.src = ""; - audioElementRevokersRef.current.get(region.id)?.(); - audioElementRevokersRef.current.delete(region.id); - audioElementResourcesRef.current.set(region.id, region.audioPath); + audioElementRevokersRef.current.get(track.id)?.(); + audioElementRevokersRef.current.delete(track.id); + audioElementResourcesRef.current.set(track.id, track.sourceRef.path); void (async () => { - const resolved = await resolveMediaElementSource(region.audioPath); - const latestAudio = existing.get(region.id); + const resolved = await resolveMediaElementSource(track.sourceRef.path); + const latestAudio = existing.get(track.id); if ( cancelled || latestAudio !== audio || - audioElementResourcesRef.current.get(region.id) !== region.audioPath + audioElementResourcesRef.current.get(track.id) !== track.sourceRef.path ) { resolved.revoke(); return; } - audioElementRevokersRef.current.set(region.id, resolved.revoke); + audioElementRevokersRef.current.set(track.id, resolved.revoke); latestAudio.src = resolved.src; })(); } - audio.volume = Math.max(0, Math.min(1, region.volume * previewVolume)); + audio.volume = Math.max(0, Math.min(1, track.gain * previewVolume)); } return () => { cancelled = true; }; - }, [audioRegions, previewVolume]); + }, [previewVolume, resolvedUserTracks]); useEffect(() => { let cancelled = false; const existing = sourceAudioElementsRef.current; - const currentIds = new Set(previewSourceAudioFallbackPaths); + const currentIds = new Set(resolvedSourceTracks.map((track) => track.sourceRef.path)); for (const [id, audio] of existing) { if (!currentIds.has(id)) { @@ -158,7 +177,8 @@ export function useAudioPreviewSync({ } } - for (const audioPath of previewSourceAudioFallbackPaths) { + for (const track of resolvedSourceTracks) { + const audioPath = track.sourceRef.path; let audio = existing.get(audioPath); if (!audio) { audio = new Audio(); @@ -237,7 +257,7 @@ export function useAudioPreviewSync({ : Math.max(0, Math.min(1, previewVolume)); } - if (previewSourceAudioFallbackPaths.length === 0) { + if (resolvedSourceTracks.length === 0) { lastSourceAudioSyncTimeRef.current = null; } @@ -248,7 +268,7 @@ export function useAudioPreviewSync({ getSourceTrackPreviewGain, isCurrentClipMuted, onSourceFallbackLoadError, - previewSourceAudioFallbackPaths, + resolvedSourceTracks, previewVolume, ]); @@ -303,15 +323,17 @@ export function useAudioPreviewSync({ ); const targetPlaybackRate = activeSpeedRegion ? activeSpeedRegion.speed : 1; - for (const region of audioRegions) { - const audio = audioElementsRef.current.get(region.id); + for (const track of resolvedUserTracks) { + const audio = audioElementsRef.current.get(track.id); if (!audio) continue; - const isInRegion = currentTimeMs >= region.startMs && currentTimeMs < region.endMs; + const startMs = track.timelineBinding.startMs; + const endMs = track.timelineBinding.endMs; + const isInRegion = currentTimeMs >= startMs && currentTimeMs < endMs; if (isPlaying && isInRegion) { enablePitchPreservingPlayback(audio); - const audioOffset = (currentTimeMs - region.startMs) / 1000; + const audioOffset = (currentTimeMs - startMs) / 1000; if (Math.abs(audio.currentTime - audioOffset) > 0.2) { audio.currentTime = audioOffset; } @@ -330,10 +352,10 @@ export function useAudioPreviewSync({ audio.pause(); } } - }, [audioRegions, timelineTime, effectiveSpeedRegions, isPlaying]); + }, [effectiveSpeedRegions, isPlaying, resolvedUserTracks, timelineTime]); useEffect(() => { - if (previewSourceAudioFallbackPaths.length === 0) { + if (resolvedSourceTracks.length === 0) { lastSourceAudioSyncTimeRef.current = null; return; } @@ -418,12 +440,12 @@ export function useAudioPreviewSync({ isCurrentClipMuted, isPlaying, previewVolume, - previewSourceAudioFallbackPaths, + resolvedSourceTracks, sourceAudioFallbackStartDelayMsByPath, ]); useEffect(() => { - if (!isPlaying || previewSourceAudioFallbackPaths.length === 0) { + if (!isPlaying || resolvedSourceTracks.length === 0) { return; } void ensureSourceAudioRunning().then(() => { @@ -433,5 +455,5 @@ export function useAudioPreviewSync({ } } }); - }, [isPlaying, previewSourceAudioFallbackPaths]); + }, [isPlaying, resolvedSourceTracks.length]); } diff --git a/src/components/video-editor/audio/useSourceAudioTrackSettings.ts b/src/components/video-editor/audio/useSourceAudioTrackSettings.ts index 34ed0ee3..54b2d751 100644 --- a/src/components/video-editor/audio/useSourceAudioTrackSettings.ts +++ b/src/components/video-editor/audio/useSourceAudioTrackSettings.ts @@ -24,6 +24,19 @@ export interface UseSourceAudioTrackSettingsResult { onSelectedClipSourceAudioTrackNormalizeChange: (id: string, normalize: boolean) => void; } +function isSameTrackMeta(left: SourceAudioTrackMeta, right: SourceAudioTrackMeta): boolean { + if (left.length !== right.length) return false; + for (let index = 0; index < left.length; index += 1) { + const leftTrack = left[index]; + const rightTrack = right[index]; + if (!leftTrack || !rightTrack) return false; + if (leftTrack.id !== rightTrack.id || leftTrack.label !== rightTrack.label) { + return false; + } + } + return true; +} + export function useSourceAudioTrackSettings({ selectedClipId, activeClipId, @@ -55,13 +68,31 @@ export function useSourceAudioTrackSettings({ }, [defaultSourceAudioTrackSettings, selectedClipId, sourceAudioTrackSettingsByClip]); const onSourceAudioTracksMetaChange = useCallback((tracks: SourceAudioTrackMeta) => { - setSourceAudioTrackMeta(tracks); + setSourceAudioTrackMeta((prev) => (isSameTrackMeta(prev, tracks) ? prev : tracks)); setDefaultSourceAudioTrackSettings((prev) => { const next: SourceAudioTrackSettings = {}; for (const track of tracks) { next[track.id] = prev[track.id] ?? { volume: 1, normalize: false }; } - return next; + const prevKeys = Object.keys(prev); + const nextKeys = Object.keys(next); + if (prevKeys.length !== nextKeys.length) { + return next; + } + for (const key of nextKeys) { + const prevSetting = prev[key]; + const nextSetting = next[key]; + if (!prevSetting || !nextSetting) { + return next; + } + if ( + prevSetting.volume !== nextSetting.volume || + prevSetting.normalize !== nextSetting.normalize + ) { + return next; + } + } + return prev; }); }, []); diff --git a/src/components/video-editor/projectPersistence.ts b/src/components/video-editor/projectPersistence.ts index 49f74348..4ba24920 100644 --- a/src/components/video-editor/projectPersistence.ts +++ b/src/components/video-editor/projectPersistence.ts @@ -654,16 +654,17 @@ export function normalizeProjectEditor(editor: Partial): Pro const startMs = Math.max(0, Math.min(rawStart, rawEnd)); const endMs = Math.max(startMs + 1, rawEnd); - return { - id: region.id, - startMs, - endMs, - audioPath: typeof region.audioPath === "string" ? region.audioPath : "", - volume: isFiniteNumber(region.volume) ? clamp(region.volume, 0, 1) : 1, - trackIndex: isFiniteNumber(region.trackIndex) - ? Math.max(0, Math.floor(region.trackIndex)) - : 0, - }; + return { + id: region.id, + startMs, + endMs, + audioPath: typeof region.audioPath === "string" ? region.audioPath : "", + volume: isFiniteNumber(region.volume) ? clamp(region.volume, 0, 1) : 1, + normalize: Boolean(region.normalize), + trackIndex: isFiniteNumber(region.trackIndex) + ? Math.max(0, Math.floor(region.trackIndex)) + : 0, + }; }) : []; diff --git a/src/components/video-editor/timeline/components/viewport/TimelineCanvas.tsx b/src/components/video-editor/timeline/components/viewport/TimelineCanvas.tsx index a1617779..806593c5 100644 --- a/src/components/video-editor/timeline/components/viewport/TimelineCanvas.tsx +++ b/src/components/video-editor/timeline/components/viewport/TimelineCanvas.tsx @@ -263,16 +263,18 @@ function AudioItemWithWaveform({ return { start: 0, end: duration }; }, [waveformSpan.end, waveformSpan.start]); return ( - + {item.label} ); diff --git a/src/components/video-editor/timeline/core/timelineTypes.ts b/src/components/video-editor/timeline/core/timelineTypes.ts index 1fc9a66a..3464b8ad 100644 --- a/src/components/video-editor/timeline/core/timelineTypes.ts +++ b/src/components/video-editor/timeline/core/timelineTypes.ts @@ -33,6 +33,8 @@ export interface TimelineRenderItem { span: Span; label: string; audioPath?: string; + audioGain?: number; + audioNormalize?: boolean; zoomDepth?: number; zoomMode?: ZoomMode; speedValue?: number; diff --git a/src/components/video-editor/timeline/model/timelineModel.ts b/src/components/video-editor/timeline/model/timelineModel.ts index 600859a5..05073c97 100644 --- a/src/components/video-editor/timeline/model/timelineModel.ts +++ b/src/components/video-editor/timeline/model/timelineModel.ts @@ -70,6 +70,8 @@ export function buildTimelineItems(params: { span: { start: region.startMs, end: region.endMs }, label: getAudioLabel(region), audioPath: region.audioPath, + audioGain: region.volume, + audioNormalize: Boolean(region.normalize), variant: "audio", })); diff --git a/src/components/video-editor/types.ts b/src/components/video-editor/types.ts index 838b4646..3e18f720 100644 --- a/src/components/video-editor/types.ts +++ b/src/components/video-editor/types.ts @@ -480,6 +480,7 @@ export interface AudioRegion { endMs: number; audioPath: string; volume: number; + normalize?: boolean; trackIndex?: number; } diff --git a/src/lib/exporter/audioEncoder.ts b/src/lib/exporter/audioEncoder.ts index c6f014f8..9e077f25 100644 --- a/src/lib/exporter/audioEncoder.ts +++ b/src/lib/exporter/audioEncoder.ts @@ -6,13 +6,14 @@ import type { SourceAudioTrackSettings, TrimRegion, } from "@/components/video-editor/types"; +import { + buildResolvedAudioPlan, + getSourceTrackIdFromPath, +} from "@/lib/exporter/audioRoutingEngine"; import { estimateCompanionAudioStartDelaySeconds } from "@/lib/mediaTiming"; import { resolveMediaElementSource } from "./localMediaSource"; import type { VideoMuxer } from "./muxer"; -import { - getSourceTrackIdFromPath, - resolveSourceTrackRoutingPolicy, -} from "./sourceTrackRoutingPolicy"; +import { resolveSourceTrackRoutingPolicy } from "./sourceTrackRoutingPolicy"; const AUDIO_BITRATE = 128_000; const DECODE_BACKPRESSURE_LIMIT = 20; @@ -22,6 +23,7 @@ const MP4_AUDIO_CODEC = "mp4a.40.2"; const OFFLINE_AUDIO_SAMPLE_RATE = 48_000; const OFFLINE_ENCODE_CHUNK_FRAMES = 1024; const OFFLINE_CHUNK_DURATION_SEC = 30; +const USER_AUDIO_NORMALIZE_GAIN = 1.35; interface TimelineSlice { sourceStartMs: number; @@ -600,13 +602,28 @@ export class AudioProcessor { if (this.cancelled) throw new Error("Export cancelled"); this.onProgress?.(0); - const routingPolicy = resolveSourceTrackRoutingPolicy( - videoUrl, + const resolvedPlan = buildResolvedAudioPlan({ + videoResource: videoUrl, sourceAudioFallbackPaths, - ); + audioRegions, + sourceTrackGainById: { + mic: Math.max(0, Math.min(2, sourceAudioTrackSettings?.mic?.volume ?? 1)), + system: Math.max(0, Math.min(2, sourceAudioTrackSettings?.system?.volume ?? 1)), + mixed: Math.max(0, Math.min(2, sourceAudioTrackSettings?.mixed?.volume ?? 1)), + }, + embeddedGain: Math.max( + 0, + Math.min( + 2, + sourceAudioTrackSettings?.mixed?.volume ?? + sourceAudioTrackSettings?.system?.volume ?? + 1, + ), + ), + }); // Decode embedded source audio separately from companion sidecars. - const mainBuffer = routingPolicy.includeEmbeddedInExport + const mainBuffer = resolvedPlan.includeEmbeddedInExport ? await this.decodeAudioFromUrl(videoUrl) : null; const mainBufferGainSettings = @@ -622,8 +639,8 @@ export class AudioProcessor { []; const refDuration = mainBuffer?.duration ?? - (routingPolicy.playbackPaths.length > 0 ? await this.getMediaDurationSec(videoUrl) : 0); - for (const audioPath of routingPolicy.playbackPaths) { + (resolvedPlan.playbackPaths.length > 0 ? await this.getMediaDurationSec(videoUrl) : 0); + for (const audioPath of resolvedPlan.playbackPaths) { if (this.cancelled) throw new Error("Export cancelled"); const buffer = await this.decodeAudioFromUrl(audioPath); if (!buffer) continue; @@ -662,7 +679,7 @@ export class AudioProcessor { let sourceDurationSec: number; if (mainBufferEntry?.buffer) { sourceDurationSec = mainBufferEntry.buffer.duration; - } else if (routingPolicy.playbackPaths.length > 0 || regionEntries.length > 0) { + } else if (resolvedPlan.playbackPaths.length > 0 || regionEntries.length > 0) { sourceDurationSec = await this.getMediaDurationSec(videoUrl); } else { sourceDurationSec = primaryBuffer?.duration ?? 0; @@ -892,7 +909,8 @@ export class AudioProcessor { if (duration <= 0.001) return; const gainNode = ctx.createGain(); - gainNode.gain.value = Math.max(0, Math.min(1, region.volume)); + const normalizeGain = region.normalize ? USER_AUDIO_NORMALIZE_GAIN : 1; + gainNode.gain.value = Math.max(0, Math.min(1, region.volume * normalizeGain)); gainNode.connect(ctx.destination); const source = ctx.createBufferSource(); diff --git a/src/lib/exporter/audioRoutingEngine.ts b/src/lib/exporter/audioRoutingEngine.ts new file mode 100644 index 00000000..a9a1a002 --- /dev/null +++ b/src/lib/exporter/audioRoutingEngine.ts @@ -0,0 +1,129 @@ +import type { AudioRegion } from "@/components/video-editor/types"; +import { resolveSourceAudioFallbackPaths } from "./sourceAudioFallback"; + +export type SourceTrackId = "mic" | "system" | "mixed"; +export type ResolvedAudioTrackKind = "user" | "system" | "mic" | "mixed" | "embedded"; +const USER_AUDIO_NORMALIZE_GAIN = 1.35; + +export interface ResolvedAudioTrack { + id: string; + kind: ResolvedAudioTrackKind; + sourceRef: { + path: string; + startDelayMs: number; + }; + gain: number; + timelineBinding: { + startMs: number; + endMs: number; + }; +} + +export interface ResolvedAudioPlan { + hasEmbeddedSourceAudio: boolean; + pathsByTrack: Partial>; + playbackPaths: string[]; + muteEmbeddedPreview: boolean; + includeEmbeddedInExport: boolean; + tracks: ResolvedAudioTrack[]; + masterGain: number; +} + +export function getSourceTrackIdFromPath(audioPath: string): SourceTrackId { + const normalized = audioPath.toLowerCase(); + if (normalized.includes(".mic.")) return "mic"; + if (normalized.includes(".system.")) return "system"; + return "mixed"; +} + +function clampGain(value: number, max: number) { + if (!Number.isFinite(value)) return 1; + return Math.max(0, Math.min(max, value)); +} + +export function buildResolvedAudioPlan(input: { + videoResource: string | null | undefined; + sourceAudioFallbackPaths: string[] | null | undefined; + audioRegions?: AudioRegion[]; + sourceTrackGainById?: Partial>; + embeddedGain?: number; + masterGain?: number; +}): ResolvedAudioPlan { + const { hasEmbeddedSourceAudio, externalAudioPaths } = resolveSourceAudioFallbackPaths( + input.videoResource, + input.sourceAudioFallbackPaths, + ); + + const pathsByTrack: Partial> = {}; + for (const path of externalAudioPaths) { + const trackId = getSourceTrackIdFromPath(path); + if (!pathsByTrack[trackId]) { + pathsByTrack[trackId] = path; + } + } + + const hasDedicatedTracks = Boolean(pathsByTrack.system || pathsByTrack.mic); + const playbackPaths: string[] = []; + if (pathsByTrack.system) playbackPaths.push(pathsByTrack.system); + if (pathsByTrack.mic) playbackPaths.push(pathsByTrack.mic); + if (!hasDedicatedTracks && pathsByTrack.mixed) playbackPaths.push(pathsByTrack.mixed); + + const includeEmbeddedInExport = !pathsByTrack.system && !pathsByTrack.mixed; + const resolvedRegions = (input.audioRegions ?? []).slice().sort((a, b) => a.startMs - b.startMs); + const tracks: ResolvedAudioTrack[] = resolvedRegions.map((region) => ({ + id: `user:${region.id}`, + kind: "user", + sourceRef: { + path: region.audioPath, + startDelayMs: 0, + }, + gain: clampGain(region.volume * (region.normalize ? USER_AUDIO_NORMALIZE_GAIN : 1), 1), + timelineBinding: { + startMs: Math.max(0, region.startMs), + endMs: Math.max(0, region.endMs), + }, + })); + + for (const audioPath of playbackPaths) { + const trackId = getSourceTrackIdFromPath(audioPath); + tracks.push({ + id: `${trackId}:${audioPath}`, + kind: trackId, + sourceRef: { + path: audioPath, + startDelayMs: 0, + }, + gain: clampGain(input.sourceTrackGainById?.[trackId] ?? 1, 2), + timelineBinding: { + startMs: 0, + endMs: Number.POSITIVE_INFINITY, + }, + }); + } + + if (hasEmbeddedSourceAudio && input.videoResource) { + tracks.push({ + id: `embedded:${input.videoResource}`, + kind: "embedded", + sourceRef: { + path: input.videoResource, + startDelayMs: 0, + }, + gain: clampGain(input.embeddedGain ?? input.sourceTrackGainById?.mixed ?? 1, 2), + timelineBinding: { + startMs: 0, + endMs: Number.POSITIVE_INFINITY, + }, + }); + } + + return { + hasEmbeddedSourceAudio, + pathsByTrack, + playbackPaths, + muteEmbeddedPreview: hasDedicatedTracks, + includeEmbeddedInExport, + tracks, + masterGain: clampGain(input.masterGain ?? 1, 1), + }; +} diff --git a/src/lib/exporter/sourceTrackRoutingPolicy.ts b/src/lib/exporter/sourceTrackRoutingPolicy.ts index 67601362..f09e00ea 100644 --- a/src/lib/exporter/sourceTrackRoutingPolicy.ts +++ b/src/lib/exporter/sourceTrackRoutingPolicy.ts @@ -1,13 +1,10 @@ -import { resolveSourceAudioFallbackPaths } from "./sourceAudioFallback"; +import { + buildResolvedAudioPlan, + getSourceTrackIdFromPath, + type SourceTrackId, +} from "./audioRoutingEngine"; -export type SourceTrackId = "mic" | "system" | "mixed"; - -export function getSourceTrackIdFromPath(audioPath: string): SourceTrackId { - const normalized = audioPath.toLowerCase(); - if (normalized.includes(".mic.")) return "mic"; - if (normalized.includes(".system.")) return "system"; - return "mixed"; -} +export { getSourceTrackIdFromPath, type SourceTrackId }; export interface SourceTrackRoutingPolicy { hasEmbeddedSourceAudio: boolean; @@ -21,30 +18,16 @@ export function resolveSourceTrackRoutingPolicy( videoResource: string | null | undefined, sourceAudioFallbackPaths: string[] | null | undefined, ): SourceTrackRoutingPolicy { - const { hasEmbeddedSourceAudio, externalAudioPaths } = resolveSourceAudioFallbackPaths( + const plan = buildResolvedAudioPlan({ videoResource, sourceAudioFallbackPaths, - ); - - const pathsByTrack: Partial> = {}; - for (const path of externalAudioPaths) { - const trackId = getSourceTrackIdFromPath(path); - if (!pathsByTrack[trackId]) { - pathsByTrack[trackId] = path; - } - } - - const hasDedicatedTracks = Boolean(pathsByTrack.system || pathsByTrack.mic); - const playbackPaths: string[] = []; - if (pathsByTrack.system) playbackPaths.push(pathsByTrack.system); - if (pathsByTrack.mic) playbackPaths.push(pathsByTrack.mic); - if (!hasDedicatedTracks && pathsByTrack.mixed) playbackPaths.push(pathsByTrack.mixed); + }); return { - hasEmbeddedSourceAudio, - pathsByTrack, - playbackPaths, - muteEmbeddedPreview: hasDedicatedTracks, - includeEmbeddedInExport: !pathsByTrack.system && !pathsByTrack.mixed, + hasEmbeddedSourceAudio: plan.hasEmbeddedSourceAudio, + pathsByTrack: plan.pathsByTrack, + playbackPaths: plan.playbackPaths, + muteEmbeddedPreview: plan.muteEmbeddedPreview, + includeEmbeddedInExport: plan.includeEmbeddedInExport, }; } From ef993a09d00aa9b544deeeec5804569720edae69 Mon Sep 17 00:00:00 2001 From: Alan Trebugeais Date: Sat, 9 May 2026 19:24:07 +0200 Subject: [PATCH 12/25] fix: audioRegions are now allowed (reloading a project with a user audio would not work. --- electron/ipc/project/manager.ts | 8 ++++++ src/components/video-editor/VideoEditor.tsx | 2 +- .../video-editor/audio/audioTypes.ts | 25 +++++++++++++++++++ .../video-editor/audio/sourceAudioTracks.ts | 5 ---- .../audio/useClipAudioSettingsController.ts | 6 ++--- .../audio/useSourceAudioFallback.ts | 2 +- .../audio/useSourceAudioTrackSettings.ts | 7 +++--- .../video-editor/audio/useVideoEditorAudio.ts | 2 +- .../video-editor/projectPersistence.ts | 2 +- .../video-editor/timeline/TimelineEditor.tsx | 16 +++++++----- .../components/viewport/TimelineCanvas.tsx | 14 +++++++---- src/components/video-editor/types.ts | 8 +----- src/lib/exporter/sourceTrackRoutingPolicy.ts | 3 --- 13 files changed, 64 insertions(+), 36 deletions(-) create mode 100644 src/components/video-editor/audio/audioTypes.ts delete mode 100644 src/components/video-editor/audio/sourceAudioTracks.ts diff --git a/electron/ipc/project/manager.ts b/electron/ipc/project/manager.ts index 404654b1..2f6e4fc2 100644 --- a/electron/ipc/project/manager.ts +++ b/electron/ipc/project/manager.ts @@ -428,6 +428,7 @@ export async function loadProjectFromPath(projectPath: string) { const projectObj = project as Record; const editorObj = projectObj?.editor as Record | undefined; const audioTracks = editorObj?.audioTracks as { sourcePath?: unknown }[] | undefined; + const audioRegions = editorObj?.audioRegions as { audioPath?: unknown }[] | undefined; const approvedProjectPaths: Array = [ mediaSources.videoPath, mediaSources.webcamPath, @@ -439,6 +440,13 @@ export async function loadProjectFromPath(projectPath: string) { } } } + if (Array.isArray(audioRegions)) { + for (const region of audioRegions) { + if (typeof region?.audioPath === "string") { + approvedProjectPaths.push(region.audioPath); + } + } + } await replaceApprovedSessionLocalReadPaths(approvedProjectPaths); await rememberRecentProject(normalizedPath); diff --git a/src/components/video-editor/VideoEditor.tsx b/src/components/video-editor/VideoEditor.tsx index 83992392..08988c6d 100644 --- a/src/components/video-editor/VideoEditor.tsx +++ b/src/components/video-editor/VideoEditor.tsx @@ -149,6 +149,7 @@ import { } from "./TutorialHelp"; import TimelineEditor, { type TimelineEditorHandle } from "./timeline/TimelineEditor"; import { normalizeCursorTelemetry } from "./timeline/zoomSuggestionUtils"; +import type { SourceAudioTrackSettings } from "@/components/video-editor/audio/audioTypes"; import { type AnnotationRegion, type AudioRegion, @@ -197,7 +198,6 @@ import { type ZoomMotionBlurTuning, type ZoomRegion, type ZoomTransitionEasing, - type SourceAudioTrackSettings, } from "./types"; import VideoPlayback, { VideoPlaybackRef } from "./VideoPlayback"; import { diff --git a/src/components/video-editor/audio/audioTypes.ts b/src/components/video-editor/audio/audioTypes.ts new file mode 100644 index 00000000..d20c224f --- /dev/null +++ b/src/components/video-editor/audio/audioTypes.ts @@ -0,0 +1,25 @@ +import type { AudioPeaksData } from "../timeline/core/timelineTypes"; + +export type SourceAudioTrackId = "mixed" | "system" | "mic" | (string & {}); + +export interface SourceAudioTrackSetting { + volume: number; + normalize: boolean; +} + +export type SourceAudioTrackSettings = Record; + +export interface SourceAudioTrackMetaItem { + id: SourceAudioTrackId; + label: string; +} + +export type SourceAudioTrackMeta = SourceAudioTrackMetaItem[]; + +export interface SourceAudioTrackWithPeaks extends SourceAudioTrackMetaItem { + peaks: AudioPeaksData; +} + +export const SOURCE_AUDIO_FALLBACK_TOAST_ID = "source-audio-fallback-error"; +export const SOURCE_AUDIO_NORMALIZE_GAIN = 1.35; + diff --git a/src/components/video-editor/audio/sourceAudioTracks.ts b/src/components/video-editor/audio/sourceAudioTracks.ts deleted file mode 100644 index b3ea2cee..00000000 --- a/src/components/video-editor/audio/sourceAudioTracks.ts +++ /dev/null @@ -1,5 +0,0 @@ -import { getSourceTrackIdFromPath } from "@/lib/exporter/sourceTrackRoutingPolicy"; - -export const SOURCE_AUDIO_FALLBACK_TOAST_ID = "source-audio-fallback-error"; -export const SOURCE_AUDIO_NORMALIZE_GAIN = 1.35; -export { getSourceTrackIdFromPath }; diff --git a/src/components/video-editor/audio/useClipAudioSettingsController.ts b/src/components/video-editor/audio/useClipAudioSettingsController.ts index 8c79ac09..08ad0d5e 100644 --- a/src/components/video-editor/audio/useClipAudioSettingsController.ts +++ b/src/components/video-editor/audio/useClipAudioSettingsController.ts @@ -1,10 +1,10 @@ import React, { useCallback, useMemo } from "react"; import { SOURCE_AUDIO_NORMALIZE_GAIN, - getSourceTrackIdFromPath, -} from "./sourceAudioTracks"; + type SourceAudioTrackSettings, +} from "@/components/video-editor/audio/audioTypes"; import { useSourceAudioTrackSettings } from "./useSourceAudioTrackSettings"; -import { SourceAudioTrackSettings } from "../types"; +import { getSourceTrackIdFromPath } from "@/lib/exporter/audioRoutingEngine"; interface UseClipAudioSettingsControllerParams { selectedClipId: string | null; diff --git a/src/components/video-editor/audio/useSourceAudioFallback.ts b/src/components/video-editor/audio/useSourceAudioFallback.ts index 64b4caa8..69be8c24 100644 --- a/src/components/video-editor/audio/useSourceAudioFallback.ts +++ b/src/components/video-editor/audio/useSourceAudioFallback.ts @@ -1,6 +1,6 @@ import { useEffect, useState } from "react"; import { toast } from "sonner"; -import { SOURCE_AUDIO_FALLBACK_TOAST_ID } from "./sourceAudioTracks"; +import { SOURCE_AUDIO_FALLBACK_TOAST_ID } from "@/components/video-editor/audio/audioTypes"; interface UseSourceAudioFallbackParams { currentSourcePath: string | null; diff --git a/src/components/video-editor/audio/useSourceAudioTrackSettings.ts b/src/components/video-editor/audio/useSourceAudioTrackSettings.ts index 54b2d751..2e776eb2 100644 --- a/src/components/video-editor/audio/useSourceAudioTrackSettings.ts +++ b/src/components/video-editor/audio/useSourceAudioTrackSettings.ts @@ -1,7 +1,8 @@ import React, { useCallback, useMemo, useState } from "react"; -import type { SourceAudioTrackSettings } from "../types"; - -export type SourceAudioTrackMeta = Array<{ id: string; label: string }>; +import type { + SourceAudioTrackMeta, + SourceAudioTrackSettings, +} from "@/components/video-editor/audio/audioTypes"; interface UseSourceAudioTrackSettingsParams { selectedClipId: string | null; diff --git a/src/components/video-editor/audio/useVideoEditorAudio.ts b/src/components/video-editor/audio/useVideoEditorAudio.ts index d992e871..a82ea941 100644 --- a/src/components/video-editor/audio/useVideoEditorAudio.ts +++ b/src/components/video-editor/audio/useVideoEditorAudio.ts @@ -3,9 +3,9 @@ import { resolveSourceTrackRoutingPolicy } from "@/lib/exporter/sourceTrackRouti import type { AudioRegion, ClipRegion, - SourceAudioTrackSettings, SpeedRegion, } from "../types"; +import type { SourceAudioTrackSettings } from "@/components/video-editor/audio/audioTypes"; import { getActiveClipIdAtSourceTime, isClipMutedById } from "./clipAudio"; import { useAudioPreviewSync } from "./useAudioPreviewSync"; import { useClipAudioSettingsController } from "./useClipAudioSettingsController"; diff --git a/src/components/video-editor/projectPersistence.ts b/src/components/video-editor/projectPersistence.ts index 4ba24920..9adb3608 100644 --- a/src/components/video-editor/projectPersistence.ts +++ b/src/components/video-editor/projectPersistence.ts @@ -20,6 +20,7 @@ import { import { DEFAULT_WALLPAPER_PATH } from "@/lib/wallpapers"; import { ASPECT_RATIOS, type AspectRatio, isCustomAspectRatio } from "@/utils/aspectRatioUtils"; import { CURSOR_MOTION_PRESETS, resolveCursorMotionPresetId } from "./cursorMotionPresets"; +import type { SourceAudioTrackSettings } from "@/components/video-editor/audio/audioTypes"; import { type AnnotationRegion, type AudioRegion, @@ -62,7 +63,6 @@ import { DEFAULT_ZOOM_SMOOTHNESS, getDefaultCaptionFontFamily, type Padding, - SourceAudioTrackSettings, type SpeedRegion, type TrimRegion, type WebcamOverlaySettings, diff --git a/src/components/video-editor/timeline/TimelineEditor.tsx b/src/components/video-editor/timeline/TimelineEditor.tsx index 28e83a5f..df2ed47b 100644 --- a/src/components/video-editor/timeline/TimelineEditor.tsx +++ b/src/components/video-editor/timeline/TimelineEditor.tsx @@ -18,6 +18,11 @@ import { import { formatShortcut } from "@/utils/platformUtils"; import { loadEditorPreferences, saveEditorPreferences } from "../editorPreferences"; import { fromFileUrl } from "../projectPersistence"; +import type { + SourceAudioTrackMeta, + SourceAudioTrackSettings, + SourceAudioTrackWithPeaks, +} from "@/components/video-editor/audio/audioTypes"; import type { AnnotationRegion, AudioRegion, @@ -36,7 +41,6 @@ import { useTimelineEditorRuntime } from "./hooks/useTimelineEditorRuntime"; import { useTimelineRange } from "./hooks/useTimelineRange"; import TimelineCanvas from "./components/viewport/TimelineCanvas"; import TimelineToolbar from "./components/toolbar/TimelineToolbar"; -import type { AudioPeaksData } from "./core/timelineTypes"; export interface TimelineEditorProps { videoDuration: number; @@ -84,11 +88,11 @@ export interface TimelineEditorProps { hideToolbar?: boolean; showSourceAudioTrack?: boolean; onSourceAudioAvailabilityChange?: (available: boolean) => void; - sourceAudioTrackSettings?: Record; + sourceAudioTrackSettings?: SourceAudioTrackSettings; getSourceAudioTrackSettingsForClip?: ( clipId: string | null, - ) => Record; - onSourceAudioTracksMetaChange?: (tracks: Array<{ id: string; label: string }>) => void; + ) => SourceAudioTrackSettings; + onSourceAudioTracksMetaChange?: (tracks: SourceAudioTrackMeta) => void; } function extractLocalPathFromMediaServerUrl(input: string | null | undefined): string | null { @@ -288,9 +292,9 @@ const TimelineEditor = forwardRef( ); const micSidecarPeaks = useTimelineAudioPeaks(micSidecarPath); const systemSidecarPeaks = useTimelineAudioPeaks(systemSidecarPath); - const sourceAudioTracks = useMemo>(() => { + const sourceAudioTracks = useMemo(() => { if (systemSidecarPeaks || micSidecarPeaks) { - const tracks: Array<{ id: string; label: string; peaks: AudioPeaksData }> = []; + const tracks: SourceAudioTrackWithPeaks[] = []; if (systemSidecarPeaks) tracks.push({ id: "system", diff --git a/src/components/video-editor/timeline/components/viewport/TimelineCanvas.tsx b/src/components/video-editor/timeline/components/viewport/TimelineCanvas.tsx index 806593c5..4d81b323 100644 --- a/src/components/video-editor/timeline/components/viewport/TimelineCanvas.tsx +++ b/src/components/video-editor/timeline/components/viewport/TimelineCanvas.tsx @@ -11,6 +11,10 @@ import { type MouseEventHandler, } from "react"; import { cn } from "@/lib/utils"; +import type { + SourceAudioTrackSettings, + SourceAudioTrackWithPeaks, +} from "@/components/video-editor/audio/audioTypes"; import { getTimelineContentMinHeightPx, getTimelineRowsMinHeightPx, @@ -21,7 +25,7 @@ import glassStyles from "../../ItemGlass.module.css"; import Item from "../../Item"; import Row from "../../Row"; import { CLIP_ROW_ID, SOURCE_AUDIO_ROW_ID, ZOOM_ROW_ID } from "../../core/constants"; -import type { AudioPeaksData, TimelineRenderItem } from "../../core/timelineTypes"; +import type { TimelineRenderItem } from "../../core/timelineTypes"; import { getAnnotationTrackIndex, getAnnotationTrackRowId, @@ -57,10 +61,10 @@ interface TimelineCanvasProps { selectAllBlocksActive?: boolean; onClearBlockSelection?: () => void; keyframes?: { id: string; time: number }[]; - sourceAudioTracks?: Array<{ id: string; label: string; peaks: AudioPeaksData }>; + sourceAudioTracks?: SourceAudioTrackWithPeaks[]; getSourceAudioTrackSettingsForClip?: ( clipId: string | null, - ) => Record; + ) => SourceAudioTrackSettings; showSourceAudioTrack?: boolean; liveSpanPreviewById?: Record; liveHiddenItemIds?: string[]; @@ -224,10 +228,10 @@ interface TimelineCanvasRowsProps { onSelectClip?: (id: string | null) => void; onSelectAnnotation?: (id: string | null) => void; onSelectAudio?: (id: string | null) => void; - sourceAudioTracks?: Array<{ id: string; label: string; peaks: AudioPeaksData }>; + sourceAudioTracks?: SourceAudioTrackWithPeaks[]; getSourceAudioTrackSettingsForClip?: ( clipId: string | null, - ) => Record; + ) => SourceAudioTrackSettings; showSourceAudioTrack?: boolean; liveSpanPreviewById?: Record; liveHiddenItemIds?: string[]; diff --git a/src/components/video-editor/types.ts b/src/components/video-editor/types.ts index 3e18f720..e6a7c6f0 100644 --- a/src/components/video-editor/types.ts +++ b/src/components/video-editor/types.ts @@ -466,13 +466,7 @@ export const DEFAULT_PADDING: Padding = { right: 20, linked: true, }; - -export interface SourceAudioTrackSetting { - volume: number; - normalize: boolean; -} - -export type SourceAudioTrackSettings = Record; +export type { SourceAudioTrackSetting, SourceAudioTrackSettings } from "@/components/video-editor/audio/audioTypes"; export interface AudioRegion { id: string; diff --git a/src/lib/exporter/sourceTrackRoutingPolicy.ts b/src/lib/exporter/sourceTrackRoutingPolicy.ts index f09e00ea..05c9fa84 100644 --- a/src/lib/exporter/sourceTrackRoutingPolicy.ts +++ b/src/lib/exporter/sourceTrackRoutingPolicy.ts @@ -1,11 +1,8 @@ import { buildResolvedAudioPlan, - getSourceTrackIdFromPath, type SourceTrackId, } from "./audioRoutingEngine"; -export { getSourceTrackIdFromPath, type SourceTrackId }; - export interface SourceTrackRoutingPolicy { hasEmbeddedSourceAudio: boolean; pathsByTrack: Partial>; From 561090a4739293dbd97443155ff8cb555a0c4cf7 Mon Sep 17 00:00:00 2001 From: Alan Trebugeais Date: Sat, 9 May 2026 19:30:40 +0200 Subject: [PATCH 13/25] add manager tests --- electron/ipc/project/manager.test.ts | 30 ++++++++++++++++++++++++++ electron/ipc/recording/audioFilters.ts | 2 +- 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/electron/ipc/project/manager.test.ts b/electron/ipc/project/manager.test.ts index 12059f7a..3fa810da 100644 --- a/electron/ipc/project/manager.test.ts +++ b/electron/ipc/project/manager.test.ts @@ -172,4 +172,34 @@ describe("local media path policy", () => { expect(result.path).toBe(projectPath); expect(result.project).toMatchObject({ videoPath }); }); + + it("approves editor audioRegions audioPath entries when loading a project", async () => { + const downloadsPath = path.join(tempRoot, "Downloads"); + const videoPath = path.join(tempPath, "recording.mp4"); + const audioPath = path.join(downloadsPath, "music.ogg"); + const projectPath = path.join(tempPath, "recording.recordly"); + await fs.mkdir(downloadsPath, { recursive: true }); + await fs.writeFile(videoPath, "test-video"); + await fs.writeFile(audioPath, "test-audio"); + await fs.writeFile( + projectPath, + JSON.stringify({ + version: 1, + videoPath, + editor: { + audioRegions: [ + { id: "a1", startMs: 0, endMs: 1000, audioPath, volume: 1 }, + ], + }, + }), + "utf-8", + ); + + const { loadProjectFromPath, resolveApprovedLocalMediaPath } = await import("./manager"); + const resolvedAudioPath = await fs.realpath(audioPath); + + const result = await loadProjectFromPath(projectPath); + expect(result.success).toBe(true); + await expect(resolveApprovedLocalMediaPath(audioPath)).resolves.toBe(resolvedAudioPath); + }); }); diff --git a/electron/ipc/recording/audioFilters.ts b/electron/ipc/recording/audioFilters.ts index 5da2a29e..b465f5f2 100644 --- a/electron/ipc/recording/audioFilters.ts +++ b/electron/ipc/recording/audioFilters.ts @@ -32,7 +32,7 @@ export function getBrowserMicSidecarFilters(profile?: string | null) { return BROWSER_MIC_SIDECAR_FILTERS; } -export const RECORDING_AUDIO_SIDECAR_DEBUG_ENV = "RECORDLY_KEEP_RECORDING_AUDIO_SIDECARS"; +export const RECORDING_AUDIO_SIDECAR_DEBUG_ENV = "RECORDLY_KEEP_RECORDING_AUDIO_SIDECARS"; // not used yet, because we need to have seperate audio files for system and mic for each recording export function shouldKeepRecordingAudioSidecars(env: NodeJS.ProcessEnv = process.env) { const value = env[RECORDING_AUDIO_SIDECAR_DEBUG_ENV]?.trim().toLowerCase(); From 5e5dd7166e507ce452146de86e939022428a51ba Mon Sep 17 00:00:00 2001 From: Alan Trebugeais Date: Sat, 9 May 2026 19:43:54 +0200 Subject: [PATCH 14/25] fix audio exporting with microphone and system sound etc --- electron/ipc/recording/windows.ts | 1 - src/lib/exporter/audioEncoder.ts | 38 ++++++++++++++++--------- src/lib/exporter/modernVideoExporter.ts | 13 ++++++++- src/lib/exporter/videoExporter.ts | 14 ++++++++- 4 files changed, 49 insertions(+), 17 deletions(-) diff --git a/electron/ipc/recording/windows.ts b/electron/ipc/recording/windows.ts index 26e15b07..5b41fc4d 100644 --- a/electron/ipc/recording/windows.ts +++ b/electron/ipc/recording/windows.ts @@ -24,7 +24,6 @@ import { import type { AudioSyncAdjustment } from "../types"; import { moveFileWithOverwrite } from "../utils"; import { - RECORDING_AUDIO_SIDECAR_DEBUG_ENV, shouldKeepRecordingAudioSidecars, WINDOWS_NATIVE_MIC_PRE_FILTERS, } from "./audioFilters"; diff --git a/src/lib/exporter/audioEncoder.ts b/src/lib/exporter/audioEncoder.ts index 9e077f25..0e342c76 100644 --- a/src/lib/exporter/audioEncoder.ts +++ b/src/lib/exporter/audioEncoder.ts @@ -25,6 +25,18 @@ const OFFLINE_ENCODE_CHUNK_FRAMES = 1024; const OFFLINE_CHUNK_DURATION_SEC = 30; const USER_AUDIO_NORMALIZE_GAIN = 1.35; +function resolveSourceTrackGain( + sourceAudioTrackSettings: SourceAudioTrackSettings | undefined, + trackId: "mic" | "system" | "mixed", +) { + const settings = sourceAudioTrackSettings?.[trackId]; + if (!settings) { + return 1; + } + const normalizeGain = settings.normalize ? USER_AUDIO_NORMALIZE_GAIN : 1; + return Math.max(0, Math.min(2, settings.volume * normalizeGain)); +} + interface TimelineSlice { sourceStartMs: number; sourceEndMs: number; @@ -607,17 +619,19 @@ export class AudioProcessor { sourceAudioFallbackPaths, audioRegions, sourceTrackGainById: { - mic: Math.max(0, Math.min(2, sourceAudioTrackSettings?.mic?.volume ?? 1)), - system: Math.max(0, Math.min(2, sourceAudioTrackSettings?.system?.volume ?? 1)), - mixed: Math.max(0, Math.min(2, sourceAudioTrackSettings?.mixed?.volume ?? 1)), + mic: resolveSourceTrackGain(sourceAudioTrackSettings, "mic"), + system: resolveSourceTrackGain(sourceAudioTrackSettings, "system"), + mixed: resolveSourceTrackGain(sourceAudioTrackSettings, "mixed"), }, embeddedGain: Math.max( 0, Math.min( 2, - sourceAudioTrackSettings?.mixed?.volume ?? - sourceAudioTrackSettings?.system?.volume ?? - 1, + sourceAudioTrackSettings?.mixed + ? resolveSourceTrackGain(sourceAudioTrackSettings, "mixed") + : sourceAudioTrackSettings?.system + ? resolveSourceTrackGain(sourceAudioTrackSettings, "system") + : 1, ), ), }); @@ -626,11 +640,7 @@ export class AudioProcessor { const mainBuffer = resolvedPlan.includeEmbeddedInExport ? await this.decodeAudioFromUrl(videoUrl) : null; - const mainBufferGainSettings = - sourceAudioTrackSettings?.mixed ?? sourceAudioTrackSettings?.system ?? null; - const mainBufferGain = mainBufferGainSettings - ? Math.max(0, Math.min(2, mainBufferGainSettings.volume)) - : 1; + const mainBufferGain = resolveSourceTrackGain(sourceAudioTrackSettings, "mixed"); const mainBufferEntry = mainBuffer ? { buffer: mainBuffer, gain: mainBufferGain } : null; if (this.cancelled) throw new Error("Export cancelled"); @@ -647,9 +657,9 @@ export class AudioProcessor { companionEntries.push({ buffer, - gain: Math.max( - 0, - Math.min(2, sourceAudioTrackSettings?.[getSourceTrackIdFromPath(audioPath)]?.volume ?? 1), + gain: resolveSourceTrackGain( + sourceAudioTrackSettings, + getSourceTrackIdFromPath(audioPath), ), startDelaySec: estimateCompanionAudioStartDelaySeconds( refDuration, diff --git a/src/lib/exporter/modernVideoExporter.ts b/src/lib/exporter/modernVideoExporter.ts index 424df4a0..3967d35a 100644 --- a/src/lib/exporter/modernVideoExporter.ts +++ b/src/lib/exporter/modernVideoExporter.ts @@ -169,6 +169,16 @@ type NativeAudioPlan = }; const FILTERGRAPH_FALLBACK_AUDIO_SAMPLE_RATE = 48_000; + +function hasNonDefaultSourceTrackSettings(sourceAudioTrackSettings?: SourceAudioTrackSettings) { + if (!sourceAudioTrackSettings) { + return false; + } + return Object.values(sourceAudioTrackSettings).some( + (settings) => + Math.abs((settings?.volume ?? 1) - 1) > 0.0005 || Boolean(settings?.normalize), + ); +} const MIN_NATIVE_STATIC_LAYOUT_SPEED = 0.25; const MAX_NATIVE_STATIC_LAYOUT_SPEED = 30; @@ -1184,7 +1194,8 @@ export class ModernVideoExporter { speedRegions.length > 0 || audioRegions.length > 0 || sourceAudioFallbackPaths.length > 1 || - hasTimedSourceAudioFallback + hasTimedSourceAudioFallback || + hasNonDefaultSourceTrackSettings(this.config.sourceAudioTrackSettings) ) { const sourceDurationMs = Math.max( 0, diff --git a/src/lib/exporter/videoExporter.ts b/src/lib/exporter/videoExporter.ts index 55b1b184..09dea96f 100644 --- a/src/lib/exporter/videoExporter.ts +++ b/src/lib/exporter/videoExporter.ts @@ -123,6 +123,16 @@ type NativeAudioPlan = const FILTERGRAPH_FALLBACK_AUDIO_SAMPLE_RATE = 48_000; +function hasNonDefaultSourceTrackSettings(sourceAudioTrackSettings?: SourceAudioTrackSettings) { + if (!sourceAudioTrackSettings) { + return false; + } + return Object.values(sourceAudioTrackSettings).some( + (settings) => + Math.abs((settings?.volume ?? 1) - 1) > 0.0005 || Boolean(settings?.normalize), + ); +} + export class VideoExporter { private config: VideoExporterConfig; private streamingDecoder: StreamingVideoDecoder | null = null; @@ -563,7 +573,8 @@ export class VideoExporter { speedRegions.length > 0 || audioRegions.length > 0 || sourceAudioFallbackPaths.length > 1 || - hasTimedSourceAudioFallback + hasTimedSourceAudioFallback || + hasNonDefaultSourceTrackSettings(this.config.sourceAudioTrackSettings) ) { const sourceDurationMs = Math.max( 0, @@ -947,6 +958,7 @@ export class VideoExporter { this.config.audioRegions, this.config.sourceAudioFallbackPaths, this.config.sourceAudioFallbackStartDelayMsByPath, + this.config.sourceAudioTrackSettings, ), "ffmpeg edited audio rendering", "audio", From c33780315abd83cc9bd5b2321ffc37373da5ae2b Mon Sep 17 00:00:00 2001 From: Alan Trebugeais Date: Sat, 9 May 2026 19:47:36 +0200 Subject: [PATCH 15/25] fix: add button to reinitialize the audio settings --- src/components/video-editor/SettingsPanel.tsx | 26 ++++++++++++++----- 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/src/components/video-editor/SettingsPanel.tsx b/src/components/video-editor/SettingsPanel.tsx index 53d42c7b..f4a8fb22 100644 --- a/src/components/video-editor/SettingsPanel.tsx +++ b/src/components/video-editor/SettingsPanel.tsx @@ -3057,9 +3057,16 @@ export function SettingsPanel({
{tSettings("audio.volumeTitle", "Audio")} - - {Math.round((selectedAudioVolume ?? 1) * 100)}% - +
{track.label} - - {Math.round(settings.volume * 100)}% - +
From 0c4771ae676e5968d18a4d980fb849d93598f239 Mon Sep 17 00:00:00 2001 From: Alan Trebugeais Date: Sat, 9 May 2026 20:18:22 +0200 Subject: [PATCH 16/25] fix: videoexports with audio (system/mic/mixed/user)... everything should work --- src/components/video-editor/SettingsPanel.tsx | 19 +++-- src/components/video-editor/VideoEditor.tsx | 1 + src/components/video-editor/timeline/Item.tsx | 9 +++ .../components/viewport/TimelineCanvas.tsx | 1 + .../timeline/core/timelineTypes.ts | 1 + .../timeline/model/timelineModel.ts | 1 + src/lib/exporter/audioEncoder.ts | 77 +++++++++++++++++-- src/lib/exporter/modernVideoExporter.ts | 6 +- src/lib/exporter/videoExporter.ts | 7 +- 9 files changed, 105 insertions(+), 17 deletions(-) diff --git a/src/components/video-editor/SettingsPanel.tsx b/src/components/video-editor/SettingsPanel.tsx index f4a8fb22..7b26b4d9 100644 --- a/src/components/video-editor/SettingsPanel.tsx +++ b/src/components/video-editor/SettingsPanel.tsx @@ -3148,14 +3148,19 @@ export function SettingsPanel({ {tSettings("audio.title", "Audio")}
- - {selectedClipMuted - ? tSettings("clip.unmuteAudio", "Unmute Audio") - : tSettings("clip.muteAudio", "Mute Audio")} - +
+ + {tSettings("clip.mute", "Mute")} + +

+ {selectedClipMuted + ? tSettings("clip.mutedState", "Audio is muted") + : tSettings("clip.unmutedState", "Audio is playing")} +

+
onClipMutedChange?.(!v)} + checked={selectedClipMuted ?? false} + onCheckedChange={(v) => onClipMutedChange?.(v)} className="data-[state=checked]:bg-[#06b6d4] scale-75" />
diff --git a/src/components/video-editor/VideoEditor.tsx b/src/components/video-editor/VideoEditor.tsx index 08988c6d..f9c75bbe 100644 --- a/src/components/video-editor/VideoEditor.tsx +++ b/src/components/video-editor/VideoEditor.tsx @@ -4417,6 +4417,7 @@ export default function VideoEditor() { cursorSway, frame, audioRegions, + clipRegions, sourceAudioFallbackPaths: audio.sourceAudioFallbackPaths, sourceAudioFallbackStartDelayMsByPath: audio.sourceAudioFallbackStartDelayMsByPath, diff --git a/src/components/video-editor/timeline/Item.tsx b/src/components/video-editor/timeline/Item.tsx index 3af2ad4d..44405905 100644 --- a/src/components/video-editor/timeline/Item.tsx +++ b/src/components/video-editor/timeline/Item.tsx @@ -5,6 +5,7 @@ import { MusicNotes as Music, MouseLeftClickIcon as PhMouseLeftClick, Scissors, + SpeakerX, MagnifyingGlassPlus as ZoomIn, } from "@phosphor-icons/react"; import type { Span } from "dnd-timeline"; @@ -31,6 +32,7 @@ interface ItemProps { waveformSegmentSpan?: Span; waveformGain?: number; waveformNormalize?: boolean; + muted?: boolean; variant?: "zoom" | "trim" | "clip" | "annotation" | "speed" | "audio"; } @@ -69,6 +71,7 @@ export default function Item({ waveformSegmentSpan, waveformGain = 1, waveformNormalize = false, + muted = false, variant = "zoom", children, }: ItemProps) { @@ -170,6 +173,12 @@ export default function Item({ className="absolute inset-0 w-full h-full pointer-events-none opacity-45" /> )} + {/* Muted overlay for source audio track items */} + {isAudio && muted && ( +
+ +
+ )} {/* Content */}
diff --git a/src/components/video-editor/timeline/components/viewport/TimelineCanvas.tsx b/src/components/video-editor/timeline/components/viewport/TimelineCanvas.tsx index 4d81b323..0c742069 100644 --- a/src/components/video-editor/timeline/components/viewport/TimelineCanvas.tsx +++ b/src/components/video-editor/timeline/components/viewport/TimelineCanvas.tsx @@ -402,6 +402,7 @@ const TimelineCanvasRows = memo(function TimelineCanvasRows({ waveformSegmentSpan={liveSpanPreviewById?.[item.id] ?? item.span} waveformGain={Math.max(0, Math.min(2, settings.volume))} waveformNormalize={Boolean(settings.normalize)} + muted={item.muted} > {track.label} diff --git a/src/components/video-editor/timeline/core/timelineTypes.ts b/src/components/video-editor/timeline/core/timelineTypes.ts index 3464b8ad..573daff4 100644 --- a/src/components/video-editor/timeline/core/timelineTypes.ts +++ b/src/components/video-editor/timeline/core/timelineTypes.ts @@ -39,6 +39,7 @@ export interface TimelineRenderItem { zoomMode?: ZoomMode; speedValue?: number; showSourceAudio?: boolean; + muted?: boolean; variant: "zoom" | "trim" | "clip" | "annotation" | "speed" | "audio"; } diff --git a/src/components/video-editor/timeline/model/timelineModel.ts b/src/components/video-editor/timeline/model/timelineModel.ts index 05073c97..70f63c4c 100644 --- a/src/components/video-editor/timeline/model/timelineModel.ts +++ b/src/components/video-editor/timeline/model/timelineModel.ts @@ -53,6 +53,7 @@ export function buildTimelineItems(params: { span: { start: region.startMs, end: region.endMs }, label: `Clip ${index + 1}`, showSourceAudio: region.showSourceAudio, + muted: Boolean(region.muted), variant: "clip", })); diff --git a/src/lib/exporter/audioEncoder.ts b/src/lib/exporter/audioEncoder.ts index 0e342c76..ec42cf83 100644 --- a/src/lib/exporter/audioEncoder.ts +++ b/src/lib/exporter/audioEncoder.ts @@ -47,6 +47,7 @@ interface PreparedOfflineRender { mainBufferEntry: { buffer: AudioBuffer; gain: number } | null; companionEntries: Array<{ buffer: AudioBuffer; startDelaySec: number; gain: number }>; regionEntries: Array<{ buffer: AudioBuffer; region: AudioRegion }>; + mutedSourceOutputRangesSec: Array<{ startSec: number; endSec: number }>; slices: TimelineSlice[]; outputDurationMs: number; numChannels: number; @@ -163,6 +164,7 @@ export class AudioProcessor { sourceAudioFallbackPaths?: string[], sourceAudioFallbackStartDelayMsByPath?: Record, sourceAudioTrackSettings?: SourceAudioTrackSettings, + clipRegions?: ClipRegion[], ): Promise { const sortedTrims = trimRegions ? [...trimRegions].sort((a, b) => a.startMs - b.startMs) @@ -206,6 +208,7 @@ export class AudioProcessor { sortedSourceAudioFallbackPaths, sourceAudioFallbackStartDelayMsByPath, sourceAudioTrackSettings, + clipRegions, muxer, ); return; @@ -238,6 +241,7 @@ export class AudioProcessor { routingPolicy.playbackPaths, sourceAudioFallbackStartDelayMsByPath, sourceAudioTrackSettings, + clipRegions, muxer, ); return; @@ -285,6 +289,7 @@ export class AudioProcessor { sourceAudioFallbackPaths?: string[], sourceAudioFallbackStartDelayMsByPath?: Record, sourceAudioTrackSettings?: SourceAudioTrackSettings, + clipRegions?: ClipRegion[], ): Promise { const sortedTrims = trimRegions ? [...trimRegions].sort((a, b) => a.startMs - b.startMs) @@ -311,6 +316,7 @@ export class AudioProcessor { sortedSourceAudioFallbackPaths, sourceAudioFallbackStartDelayMsByPath, sourceAudioTrackSettings, + clipRegions, ); return this.renderToWavBlobChunked(prepared); } @@ -587,6 +593,7 @@ export class AudioProcessor { sourceAudioFallbackPaths: string[], sourceAudioFallbackStartDelayMsByPath: Record | undefined, sourceAudioTrackSettings: SourceAudioTrackSettings | undefined, + clipRegions: ClipRegion[] | undefined, muxer: VideoMuxer, ): Promise { const prepared = await this.prepareOfflineRender( @@ -597,6 +604,7 @@ export class AudioProcessor { sourceAudioFallbackPaths, sourceAudioFallbackStartDelayMsByPath, sourceAudioTrackSettings, + clipRegions, ); if (this.cancelled) return; await this.renderAndEncodeChunked(prepared, muxer); @@ -610,6 +618,7 @@ export class AudioProcessor { sourceAudioFallbackPaths: string[], sourceAudioFallbackStartDelayMsByPath?: Record, sourceAudioTrackSettings?: SourceAudioTrackSettings, + clipRegions?: ClipRegion[], ): Promise { if (this.cancelled) throw new Error("Export cancelled"); this.onProgress?.(0); @@ -711,11 +720,24 @@ export class AudioProcessor { } const numChannels = Math.min(primaryBuffer?.numberOfChannels ?? 2, 2); + const mutedSourceOutputRangesSec = (clipRegions ?? []) + .filter( + (clip) => + Boolean(clip.muted) && + Number.isFinite(clip.startMs) && + Number.isFinite(clip.endMs) && + clip.endMs > clip.startMs, + ) + .map((clip) => ({ + startSec: Math.max(0, clip.startMs / 1000), + endSec: Math.max(0, clip.endMs / 1000), + })); return { mainBufferEntry, companionEntries, regionEntries, + mutedSourceOutputRangesSec, slices, outputDurationMs, numChannels, @@ -849,6 +871,7 @@ export class AudioProcessor { prepared.mainBufferEntry.gain, outputOffsetSec, chunkSec, + prepared.mutedSourceOutputRangesSec, ); } @@ -862,6 +885,7 @@ export class AudioProcessor { entry.gain, outputOffsetSec, chunkSec, + prepared.mutedSourceOutputRangesSec, ); } @@ -1323,6 +1347,7 @@ export class AudioProcessor { gain = 1, chunkOutputStartSec = 0, chunkDurationSec = Number.POSITIVE_INFINITY, + mutedOutputRangesSec: Array<{ startSec: number; endSec: number }> = [], ): void { let outputOffsetSec = 0; @@ -1386,15 +1411,51 @@ export class AudioProcessor { continue; } - const source = ctx.createBufferSource(); - const gainNode = ctx.createGain(); - gainNode.gain.value = Math.max(0, Math.min(2, gain)); - source.buffer = buffer; - source.playbackRate.value = slice.speed; - source.connect(gainNode); - gainNode.connect(ctx.destination); + const audibleRanges: Array<{ startSec: number; endSec: number }> = [ + { + startSec: localOutputStartSec + chunkOutputStartSec, + endSec: + localOutputStartSec + chunkOutputStartSec + effectiveSourceDurationSec / slice.speed, + }, + ]; + for (const mutedRange of mutedOutputRangesSec) { + for (let rangeIndex = audibleRanges.length - 1; rangeIndex >= 0; rangeIndex -= 1) { + const current = audibleRanges[rangeIndex]; + const overlapStart = Math.max(current.startSec, mutedRange.startSec); + const overlapEnd = Math.min(current.endSec, mutedRange.endSec); + if (overlapEnd <= overlapStart) { + continue; + } + audibleRanges.splice(rangeIndex, 1); + if (current.startSec < overlapStart) { + audibleRanges.push({ startSec: current.startSec, endSec: overlapStart }); + } + if (overlapEnd < current.endSec) { + audibleRanges.push({ startSec: overlapEnd, endSec: current.endSec }); + } + } + } - source.start(localOutputStartSec, effectiveBufferStartSec, effectiveSourceDurationSec); + for (const audibleRange of audibleRanges) { + const audibleDurationSec = audibleRange.endSec - audibleRange.startSec; + if (audibleDurationSec <= 0.001) { + continue; + } + const source = ctx.createBufferSource(); + const gainNode = ctx.createGain(); + gainNode.gain.value = Math.max(0, Math.min(2, gain)); + source.buffer = buffer; + source.playbackRate.value = slice.speed; + source.connect(gainNode); + gainNode.connect(ctx.destination); + + const sourceOffsetSec = + effectiveBufferStartSec + + (audibleRange.startSec - (localOutputStartSec + chunkOutputStartSec)) * slice.speed; + const localStartSec = audibleRange.startSec - chunkOutputStartSec; + const sourceDurationSec = audibleDurationSec * slice.speed; + source.start(localStartSec, sourceOffsetSec, sourceDurationSec); + } outputOffsetSec += sliceOutputDurationSec; } diff --git a/src/lib/exporter/modernVideoExporter.ts b/src/lib/exporter/modernVideoExporter.ts index 3967d35a..bca2c4dd 100644 --- a/src/lib/exporter/modernVideoExporter.ts +++ b/src/lib/exporter/modernVideoExporter.ts @@ -3,6 +3,7 @@ import type { AudioRegion, AutoCaptionSettings, CaptionCue, + ClipRegion, CropRegion, CursorStyle, CursorTelemetryPoint, @@ -136,6 +137,7 @@ interface VideoExporterConfig extends ExportConfig { zoomClassicMode?: boolean; frame?: string | null; audioRegions?: AudioRegion[]; + clipRegions?: ClipRegion[]; sourceAudioFallbackPaths?: string[]; sourceAudioFallbackStartDelayMsByPath?: Record; sourceAudioTrackSettings?: SourceAudioTrackSettings; @@ -1195,7 +1197,8 @@ export class ModernVideoExporter { audioRegions.length > 0 || sourceAudioFallbackPaths.length > 1 || hasTimedSourceAudioFallback || - hasNonDefaultSourceTrackSettings(this.config.sourceAudioTrackSettings) + hasNonDefaultSourceTrackSettings(this.config.sourceAudioTrackSettings) || + (this.config.clipRegions ?? []).some((clip) => Boolean(clip.muted)) ) { const sourceDurationMs = Math.max( 0, @@ -1820,6 +1823,7 @@ export class ModernVideoExporter { this.config.sourceAudioFallbackPaths, this.config.sourceAudioFallbackStartDelayMsByPath, this.config.sourceAudioTrackSettings, + this.config.clipRegions, ), description, "audio", diff --git a/src/lib/exporter/videoExporter.ts b/src/lib/exporter/videoExporter.ts index 09dea96f..b99f5f96 100644 --- a/src/lib/exporter/videoExporter.ts +++ b/src/lib/exporter/videoExporter.ts @@ -3,6 +3,7 @@ import type { AudioRegion, AutoCaptionSettings, CaptionCue, + ClipRegion, CropRegion, CursorStyle, CursorTelemetryPoint, @@ -91,6 +92,7 @@ interface VideoExporterConfig extends ExportConfig { zoomSmoothness?: number; frame?: string | null; audioRegions?: AudioRegion[]; + clipRegions?: ClipRegion[]; sourceAudioFallbackPaths?: string[]; sourceAudioFallbackStartDelayMsByPath?: Record; sourceAudioTrackSettings?: SourceAudioTrackSettings; @@ -574,7 +576,8 @@ export class VideoExporter { audioRegions.length > 0 || sourceAudioFallbackPaths.length > 1 || hasTimedSourceAudioFallback || - hasNonDefaultSourceTrackSettings(this.config.sourceAudioTrackSettings) + hasNonDefaultSourceTrackSettings(this.config.sourceAudioTrackSettings) || + (this.config.clipRegions ?? []).some((clip) => Boolean(clip.muted)) ) { const sourceDurationMs = Math.max( 0, @@ -862,6 +865,7 @@ export class VideoExporter { this.config.sourceAudioFallbackPaths, this.config.sourceAudioFallbackStartDelayMsByPath, this.config.sourceAudioTrackSettings, + this.config.clipRegions, ), "native edited audio rendering", "audio", @@ -959,6 +963,7 @@ export class VideoExporter { this.config.sourceAudioFallbackPaths, this.config.sourceAudioFallbackStartDelayMsByPath, this.config.sourceAudioTrackSettings, + this.config.clipRegions, ), "ffmpeg edited audio rendering", "audio", From 4c7964493bff869233850155e275420e128f4c9f Mon Sep 17 00:00:00 2001 From: Alan Trebugeais Date: Sat, 9 May 2026 20:24:34 +0200 Subject: [PATCH 17/25] fix locales for audio --- src/components/video-editor/SettingsPanel.tsx | 1 - src/i18n/locales/en/settings.json | 11 ++++++---- src/i18n/locales/es/settings.json | 11 ++++++---- src/i18n/locales/fr/settings.json | 11 ++++++---- src/i18n/locales/ko/settings.json | 18 ++++++++++++++--- src/i18n/locales/nl/settings.json | 18 ++++++++++++++--- src/i18n/locales/pt-BR/settings.json | 18 ++++++++++++++--- src/i18n/locales/ru/settings.json | 20 +++++++++++++++---- src/i18n/locales/zh-CN/settings.json | 13 +++++++----- src/i18n/locales/zh-TW/settings.json | 16 +++++++++++++-- 10 files changed, 104 insertions(+), 33 deletions(-) diff --git a/src/components/video-editor/SettingsPanel.tsx b/src/components/video-editor/SettingsPanel.tsx index 7b26b4d9..71e54109 100644 --- a/src/components/video-editor/SettingsPanel.tsx +++ b/src/components/video-editor/SettingsPanel.tsx @@ -3180,7 +3180,6 @@ export function SettingsPanel({ {selectedClipId && hasClipSourceAudio && - selectedClipShowSourceAudio && sourceAudioTrackMeta.length > 0 && (
{sourceAudioTrackMeta.map((track) => { diff --git a/src/i18n/locales/en/settings.json b/src/i18n/locales/en/settings.json index e8ebfebf..b9a2d00a 100644 --- a/src/i18n/locales/en/settings.json +++ b/src/i18n/locales/en/settings.json @@ -19,8 +19,9 @@ }, "clip": { "title": "Clip", - "muteAudio": "Mute Audio", - "unmuteAudio": "Unmute Audio", + "mute": "Mute", + "mutedState": "Audio is muted", + "unmutedState": "Audio is playing", "separateClipFromAudio": "Separate clip from audio", "delete": "Delete Clip" }, @@ -205,12 +206,14 @@ "starOnGithub": "Star on GitHub" }, "audio": { + "title": "Audio", + "volumeTitle": "Audio", + "volume": "Volume", + "normalize": "Normalize", "sourceTracksTitle": "Clip Source Audio", "systemLabel": "Source System", "micLabel": "Source Mic", "mixedLabel": "Source", - "volumeTitle": "Volume", - "normalize": "Normalize", "deleteRegion": "Delete Audio" } } diff --git a/src/i18n/locales/es/settings.json b/src/i18n/locales/es/settings.json index bec77412..66a05e89 100644 --- a/src/i18n/locales/es/settings.json +++ b/src/i18n/locales/es/settings.json @@ -19,8 +19,9 @@ }, "clip": { "title": "Clip", - "muteAudio": "Silenciar audio", - "unmuteAudio": "Activar sonido", + "mute": "Silenciar", + "mutedState": "El audio está silenciado", + "unmutedState": "El audio se está reproduciendo", "separateClipFromAudio": "Separar clip del audio", "delete": "Eliminar clip" }, @@ -185,12 +186,14 @@ "starOnGithub": "Estrella en GitHub" }, "audio": { + "title": "Audio", + "volumeTitle": "Audio", + "volume": "Volumen", + "normalize": "Normalizar", "sourceTracksTitle": "Audio fuente del clip", "systemLabel": "Sonido del sistema", "micLabel": "Micrófono", "mixedLabel": "Fuente", - "volumeTitle": "Volumen", - "normalize": "Normalizar", "deleteRegion": "Eliminar audio" } } diff --git a/src/i18n/locales/fr/settings.json b/src/i18n/locales/fr/settings.json index ce314c55..4f790b97 100644 --- a/src/i18n/locales/fr/settings.json +++ b/src/i18n/locales/fr/settings.json @@ -19,8 +19,9 @@ }, "clip": { "title": "Clip", - "muteAudio": "Couper le son", - "unmuteAudio": "Réactiver le son", + "mute": "Sourdine", + "mutedState": "Le son est coupé", + "unmutedState": "Le son est activé", "separateClipFromAudio": "Séparer le clip de l'audio", "delete": "Supprimer le clip" }, @@ -185,12 +186,14 @@ "starOnGithub": "Mettre une étoile sur GitHub" }, "audio": { + "title": "Audio", + "volumeTitle": "Audio", + "volume": "Volume", + "normalize": "Normaliser", "sourceTracksTitle": "Audio source du clip", "systemLabel": "Son Système", "micLabel": "Microphone", "mixedLabel": "Source", - "volumeTitle": "Volume", - "normalize": "Normaliser", "deleteRegion": "Supprimer l'audio" } } diff --git a/src/i18n/locales/ko/settings.json b/src/i18n/locales/ko/settings.json index 0adaa6c7..c528a98f 100644 --- a/src/i18n/locales/ko/settings.json +++ b/src/i18n/locales/ko/settings.json @@ -19,9 +19,10 @@ }, "clip": { "title": "클립", - "muteAudio": "오디오 음소거", - "unmuteAudio": "음소거 해제", - "separateClipFromAudio": "클립과 오디오 분리", + "mute": "음소거", + "mutedState": "오디오가 음소거됨", + "unmutedState": "오디오가 재생 중", + "separateClipFromAudio": "클립에서 오디오 분리", "delete": "클립 삭제" }, "effects": { @@ -183,5 +184,16 @@ "exportVideo": "{{format}} 내보내기", "reportBug": "버그 신고", "starOnGithub": "GitHub에서 별표 주기" + }, + "audio": { + "title": "오디오", + "volumeTitle": "오디오", + "volume": "볼륨", + "normalize": "정규화", + "sourceTracksTitle": "클립 소스 오디오", + "systemLabel": "시스템 소스", + "micLabel": "마이크 소스", + "mixedLabel": "소스", + "deleteRegion": "오디오 삭제" } } diff --git a/src/i18n/locales/nl/settings.json b/src/i18n/locales/nl/settings.json index 00d78905..d88e4278 100644 --- a/src/i18n/locales/nl/settings.json +++ b/src/i18n/locales/nl/settings.json @@ -19,9 +19,10 @@ }, "clip": { "title": "Clip", - "muteAudio": "Audio dempen", - "unmuteAudio": "Geluid inschakelen", - "separateClipFromAudio": "Clip scheiden van audio", + "mute": "Dempen", + "mutedState": "Audio is gedempt", + "unmutedState": "Audio wordt afgespeeld", + "separateClipFromAudio": "Clip van audio scheiden", "delete": "Clip verwijderen" }, "effects": { @@ -183,5 +184,16 @@ "exportVideo": "{{format}} exporteren", "reportBug": "Bug melden", "starOnGithub": "Ster op GitHub" + }, + "audio": { + "title": "Audio", + "volumeTitle": "Audio", + "volume": "Volume", + "normalize": "Normaliseren", + "sourceTracksTitle": "Bronaudio clip", + "systemLabel": "Systeemaudio", + "micLabel": "Microfoon", + "mixedLabel": "Bron", + "deleteRegion": "Audio verwijderen" } } diff --git a/src/i18n/locales/pt-BR/settings.json b/src/i18n/locales/pt-BR/settings.json index 810d7f91..a34e6e4d 100644 --- a/src/i18n/locales/pt-BR/settings.json +++ b/src/i18n/locales/pt-BR/settings.json @@ -19,9 +19,10 @@ }, "clip": { "title": "Clipe", - "muteAudio": "Silenciar áudio", - "unmuteAudio": "Ativar som", - "separateClipFromAudio": "Separar clipe do áudio", + "mute": "Mudo", + "mutedState": "O áudio está mudo", + "unmutedState": "O áudio está tocando", + "separateClipFromAudio": "Separar áudio do clipe", "delete": "Excluir clipe" }, "effects": { @@ -183,5 +184,16 @@ "exportVideo": "Exportar {{format}}", "reportBug": "Reportar bug", "starOnGithub": "Dar estrela no GitHub" + }, + "audio": { + "title": "Áudio", + "volumeTitle": "Áudio", + "volume": "Volume", + "normalize": "Normalizar", + "sourceTracksTitle": "Áudio original do clipe", + "systemLabel": "Sistema", + "micLabel": "Microfone", + "mixedLabel": "Fonte", + "deleteRegion": "Excluir áudio" } } diff --git a/src/i18n/locales/ru/settings.json b/src/i18n/locales/ru/settings.json index 6a2f052d..687f8383 100644 --- a/src/i18n/locales/ru/settings.json +++ b/src/i18n/locales/ru/settings.json @@ -19,10 +19,11 @@ }, "clip": { "title": "Клип", - "muteAudio": "Выключить звук", - "unmuteAudio": "Включить звук", - "separateClipFromAudio": "Отделить клип от аудио", - "delete": "Удалить" + "mute": "Без звука", + "mutedState": "Звук выключен", + "unmutedState": "Звук включен", + "separateClipFromAudio": "Отделить аудио от клипа", + "delete": "Удалить клип" }, "effects": { "title": "Эффекты", @@ -203,5 +204,16 @@ "exportVideo": "Экспортировать {{format}}", "reportBug": "Сообщить об ошибке", "starOnGithub": "Оценить на GitHub" + }, + "audio": { + "title": "Аудио", + "volumeTitle": "Аудио", + "volume": "Громкость", + "normalize": "Нормализовать", + "sourceTracksTitle": "Исходное аудио клипа", + "systemLabel": "Системный звук", + "micLabel": "Микрофон", + "mixedLabel": "Источник", + "deleteRegion": "Удалить аудио" } } \ No newline at end of file diff --git a/src/i18n/locales/zh-CN/settings.json b/src/i18n/locales/zh-CN/settings.json index 02c3868a..a68d2a04 100644 --- a/src/i18n/locales/zh-CN/settings.json +++ b/src/i18n/locales/zh-CN/settings.json @@ -19,8 +19,9 @@ }, "clip": { "title": "片段", - "muteAudio": "静音音频", - "unmuteAudio": "取消静音", + "mute": "静音", + "mutedState": "音频已静音", + "unmutedState": "音频正在播放", "separateClipFromAudio": "将剪辑与音频分离", "delete": "删除片段" }, @@ -200,12 +201,14 @@ "starOnGithub": "在 GitHub 上点赞" }, "audio": { - "sourceTracksTitle": "片段原始音频", + "title": "音频", + "volumeTitle": "音频", + "volume": "音量", + "normalize": "标准化", + "sourceTracksTitle": "剪辑源音频", "systemLabel": "系统声音", "micLabel": "麦克风", "mixedLabel": "来源", - "volumeTitle": "音量", - "normalize": "标准化", "deleteRegion": "删除音频" } } diff --git a/src/i18n/locales/zh-TW/settings.json b/src/i18n/locales/zh-TW/settings.json index fd9ff235..db0ea7e0 100644 --- a/src/i18n/locales/zh-TW/settings.json +++ b/src/i18n/locales/zh-TW/settings.json @@ -19,8 +19,9 @@ }, "clip": { "title": "片段", - "muteAudio": "靜音", - "unmuteAudio": "取消靜音", + "mute": "靜音", + "mutedState": "音訊已靜音", + "unmutedState": "音訊正在播放", "separateClipFromAudio": "將片段與音訊分離", "delete": "刪除片段" }, @@ -183,5 +184,16 @@ "exportVideo": "匯出 {{format}}", "reportBug": "回報錯誤", "starOnGithub": "在 GitHub 按讚" + }, + "audio": { + "title": "音訊", + "volumeTitle": "音訊", + "volume": "音量", + "normalize": "正規化", + "sourceTracksTitle": "剪輯源音訊", + "systemLabel": "系統源", + "micLabel": "麥克風源", + "mixedLabel": "源", + "deleteRegion": "刪除音訊" } } From bac15c9929ad1f2e64b76066741462ce9b9336b6 Mon Sep 17 00:00:00 2001 From: Alan Trebugeais Date: Sat, 9 May 2026 20:34:49 +0200 Subject: [PATCH 18/25] fix microphone preview --- src/lib/exporter/audioRoutingEngine.ts | 2 +- src/lib/exporter/sourceTrackRoutingPolicy.test.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/exporter/audioRoutingEngine.ts b/src/lib/exporter/audioRoutingEngine.ts index a9a1a002..cdba5777 100644 --- a/src/lib/exporter/audioRoutingEngine.ts +++ b/src/lib/exporter/audioRoutingEngine.ts @@ -121,7 +121,7 @@ export function buildResolvedAudioPlan(input: { hasEmbeddedSourceAudio, pathsByTrack, playbackPaths, - muteEmbeddedPreview: hasDedicatedTracks, + muteEmbeddedPreview: hasDedicatedTracks && !includeEmbeddedInExport, includeEmbeddedInExport, tracks, masterGain: clampGain(input.masterGain ?? 1, 1), diff --git a/src/lib/exporter/sourceTrackRoutingPolicy.test.ts b/src/lib/exporter/sourceTrackRoutingPolicy.test.ts index ca16cd9b..58743acc 100644 --- a/src/lib/exporter/sourceTrackRoutingPolicy.test.ts +++ b/src/lib/exporter/sourceTrackRoutingPolicy.test.ts @@ -35,7 +35,7 @@ describe("resolveSourceTrackRoutingPolicy", () => { ]); expect(policy.playbackPaths).toEqual(["/tmp/recording.mic.wav"]); - expect(policy.muteEmbeddedPreview).toBe(true); + expect(policy.muteEmbeddedPreview).toBe(false); expect(policy.includeEmbeddedInExport).toBe(true); }); }); From 28a8a2217aae0ca584280e62387b40c31fe3a9ca Mon Sep 17 00:00:00 2001 From: Alan Trebugeais Date: Sat, 9 May 2026 20:45:14 +0200 Subject: [PATCH 19/25] add normalized gain variable --- src/lib/exporter/audioEncoder.ts | 6 +++--- src/lib/exporter/audioRoutingEngine.ts | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/lib/exporter/audioEncoder.ts b/src/lib/exporter/audioEncoder.ts index ec42cf83..72015b5f 100644 --- a/src/lib/exporter/audioEncoder.ts +++ b/src/lib/exporter/audioEncoder.ts @@ -14,6 +14,7 @@ import { estimateCompanionAudioStartDelaySeconds } from "@/lib/mediaTiming"; import { resolveMediaElementSource } from "./localMediaSource"; import type { VideoMuxer } from "./muxer"; import { resolveSourceTrackRoutingPolicy } from "./sourceTrackRoutingPolicy"; +import { SOURCE_AUDIO_NORMALIZE_GAIN } from "@/components/video-editor/audio/audioTypes"; const AUDIO_BITRATE = 128_000; const DECODE_BACKPRESSURE_LIMIT = 20; @@ -23,7 +24,6 @@ const MP4_AUDIO_CODEC = "mp4a.40.2"; const OFFLINE_AUDIO_SAMPLE_RATE = 48_000; const OFFLINE_ENCODE_CHUNK_FRAMES = 1024; const OFFLINE_CHUNK_DURATION_SEC = 30; -const USER_AUDIO_NORMALIZE_GAIN = 1.35; function resolveSourceTrackGain( sourceAudioTrackSettings: SourceAudioTrackSettings | undefined, @@ -33,7 +33,7 @@ function resolveSourceTrackGain( if (!settings) { return 1; } - const normalizeGain = settings.normalize ? USER_AUDIO_NORMALIZE_GAIN : 1; + const normalizeGain = settings.normalize ? SOURCE_AUDIO_NORMALIZE_GAIN : 1; return Math.max(0, Math.min(2, settings.volume * normalizeGain)); } @@ -943,7 +943,7 @@ export class AudioProcessor { if (duration <= 0.001) return; const gainNode = ctx.createGain(); - const normalizeGain = region.normalize ? USER_AUDIO_NORMALIZE_GAIN : 1; + const normalizeGain = region.normalize ? SOURCE_AUDIO_NORMALIZE_GAIN : 1; gainNode.gain.value = Math.max(0, Math.min(1, region.volume * normalizeGain)); gainNode.connect(ctx.destination); diff --git a/src/lib/exporter/audioRoutingEngine.ts b/src/lib/exporter/audioRoutingEngine.ts index cdba5777..3fc3285f 100644 --- a/src/lib/exporter/audioRoutingEngine.ts +++ b/src/lib/exporter/audioRoutingEngine.ts @@ -1,9 +1,9 @@ import type { AudioRegion } from "@/components/video-editor/types"; +import { SOURCE_AUDIO_NORMALIZE_GAIN } from "@/components/video-editor/audio/audioTypes"; import { resolveSourceAudioFallbackPaths } from "./sourceAudioFallback"; export type SourceTrackId = "mic" | "system" | "mixed"; export type ResolvedAudioTrackKind = "user" | "system" | "mic" | "mixed" | "embedded"; -const USER_AUDIO_NORMALIZE_GAIN = 1.35; export interface ResolvedAudioTrack { id: string; @@ -77,7 +77,7 @@ export function buildResolvedAudioPlan(input: { path: region.audioPath, startDelayMs: 0, }, - gain: clampGain(region.volume * (region.normalize ? USER_AUDIO_NORMALIZE_GAIN : 1), 1), + gain: clampGain(region.volume * (region.normalize ? SOURCE_AUDIO_NORMALIZE_GAIN : 1), 1), timelineBinding: { startMs: Math.max(0, region.startMs), endMs: Math.max(0, region.endMs), From c9a30a2e9641a8be11931c3d179055126af895af Mon Sep 17 00:00:00 2001 From: Alan Trebugeais Date: Sat, 9 May 2026 20:48:01 +0200 Subject: [PATCH 20/25] fix bufferwebworker --- .../video-editor/audio/waveform/waveform.worker.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/components/video-editor/audio/waveform/waveform.worker.ts b/src/components/video-editor/audio/waveform/waveform.worker.ts index 7b7a5184..1db7b382 100644 --- a/src/components/video-editor/audio/waveform/waveform.worker.ts +++ b/src/components/video-editor/audio/waveform/waveform.worker.ts @@ -6,7 +6,8 @@ self.onmessage = (e: MessageEvent) => { }; if (!channelData || samples <= 0) { - self.postMessage({ requestId, peaks: new Float32Array(0) }); + const empty = new Float32Array(0); + (self as any).postMessage({ requestId, peaks: empty }, [empty.buffer]); return; } @@ -25,8 +26,9 @@ self.onmessage = (e: MessageEvent) => { result[i] = max; } - self.postMessage({ requestId, peaks: result }); + (self as any).postMessage({ requestId, peaks: result }, [result.buffer]); } catch { - self.postMessage({ requestId, peaks: new Float32Array(0) }); + const empty = new Float32Array(0); + (self as any).postMessage({ requestId, peaks: empty }, [empty.buffer]); } }; From 19069cfa32e2a90a859722d1933a0b4e902542c5 Mon Sep 17 00:00:00 2001 From: Alan Trebugeais Date: Sat, 9 May 2026 20:52:10 +0200 Subject: [PATCH 21/25] fix codeRabbits comments --- src/lib/exporter/modernVideoExporter.ts | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/src/lib/exporter/modernVideoExporter.ts b/src/lib/exporter/modernVideoExporter.ts index bca2c4dd..55f47923 100644 --- a/src/lib/exporter/modernVideoExporter.ts +++ b/src/lib/exporter/modernVideoExporter.ts @@ -767,6 +767,7 @@ export class ModernVideoExporter { this.config.sourceAudioFallbackPaths, this.config.sourceAudioFallbackStartDelayMsByPath, this.config.sourceAudioTrackSettings, + this.config.clipRegions, ), "audio processing", "audio", @@ -1218,16 +1219,20 @@ export class ModernVideoExporter { typeof primaryAudioSourceSampleRate === "number" && Number.isFinite(primaryAudioSourceSampleRate) && primaryAudioSourceSampleRate > 0; - const strategy = canUsePrimaryAudioFiltergraph - ? classifyEditedTrackStrategy({ - primaryAudioSourcePath, - sourceDurationMs, - trimRegions, - speedRegions, - audioRegions, - sourceAudioFallbackPaths, - }) - : "offline-render-fallback"; + const requiresRenderedEditedTrack = + hasNonDefaultSourceTrackSettings(this.config.sourceAudioTrackSettings) || + (this.config.clipRegions ?? []).some((clip) => Boolean(clip.muted)); + const strategy = + canUsePrimaryAudioFiltergraph && !requiresRenderedEditedTrack + ? classifyEditedTrackStrategy({ + primaryAudioSourcePath, + sourceDurationMs, + trimRegions, + speedRegions, + audioRegions, + sourceAudioFallbackPaths, + }) + : "offline-render-fallback"; if (strategy === "filtergraph-fast-path") { const audioSourcePath = primaryAudioSourcePath; From 327d80963cfcb3f3b6e2b6833026dfdf0240fe7f Mon Sep 17 00:00:00 2001 From: Alan Trebugeais Date: Sat, 9 May 2026 20:55:41 +0200 Subject: [PATCH 22/25] adressing Rabbits comments --- .../video-editor/AnnotationSettingsPanel.tsx | 4 +- .../video-editor/audio/clipAudio.ts | 9 ++- .../audio/useSourceAudioTrackSettings.ts | 4 +- .../audio/waveform/WaveformGenerator.ts | 57 ++++++++++++------- .../audio/waveform/waveform.worker.ts | 25 ++++---- 5 files changed, 60 insertions(+), 39 deletions(-) diff --git a/src/components/video-editor/AnnotationSettingsPanel.tsx b/src/components/video-editor/AnnotationSettingsPanel.tsx index be475809..db90b746 100644 --- a/src/components/video-editor/AnnotationSettingsPanel.tsx +++ b/src/components/video-editor/AnnotationSettingsPanel.tsx @@ -141,7 +141,7 @@ export function AnnotationSettingsPanel({ return (
-
+
{t("annotations.settings")} @@ -786,6 +786,7 @@ export function AnnotationSettingsPanel({
  • {t("annotations.tipCycleBackward")}
    • +
    -
    ); } diff --git a/src/components/video-editor/audio/clipAudio.ts b/src/components/video-editor/audio/clipAudio.ts index d539ad4b..c6bf7409 100644 --- a/src/components/video-editor/audio/clipAudio.ts +++ b/src/components/video-editor/audio/clipAudio.ts @@ -1,14 +1,13 @@ -import { mapSourceTimeToTimelineTime } from "../types"; +import { getClipSourceEndMs, sortClipRegions } from "../types"; import type { ClipRegion } from "../types"; export function getActiveClipIdAtSourceTime( sourceTimeSeconds: number, clipRegions: ClipRegion[], ): string | null { - const sourceMs = sourceTimeSeconds * 1000; - const timelineMs = mapSourceTimeToTimelineTime(sourceMs, clipRegions); - const activeClip = clipRegions.find( - (clip) => timelineMs >= clip.startMs && timelineMs < clip.endMs, + const sourceMs = Math.round(sourceTimeSeconds * 1000); + const activeClip = sortClipRegions(clipRegions).find( + (clip) => sourceMs >= clip.startMs && sourceMs < getClipSourceEndMs(clip), ); return activeClip?.id ?? null; } diff --git a/src/components/video-editor/audio/useSourceAudioTrackSettings.ts b/src/components/video-editor/audio/useSourceAudioTrackSettings.ts index 2e776eb2..799d6db9 100644 --- a/src/components/video-editor/audio/useSourceAudioTrackSettings.ts +++ b/src/components/video-editor/audio/useSourceAudioTrackSettings.ts @@ -120,7 +120,9 @@ export function useSourceAudioTrackSettings({ [selectedClipId]: { ...prevClip, [id]: { - volume: Math.max(0, Math.min(2, volume)), + volume: Number.isFinite(volume) + ? Math.max(0, Math.min(2, volume)) + : (prevClip[id]?.volume ?? 1), normalize: prevClip[id]?.normalize ?? false, }, }, diff --git a/src/components/video-editor/audio/waveform/WaveformGenerator.ts b/src/components/video-editor/audio/waveform/WaveformGenerator.ts index a51475d8..b44c274f 100644 --- a/src/components/video-editor/audio/waveform/WaveformGenerator.ts +++ b/src/components/video-editor/audio/waveform/WaveformGenerator.ts @@ -8,43 +8,52 @@ export class WaveformGenerator { private peaksCache = new Map(); private pending = new Map>(); private workerRequestSeq = 0; - private workerResolvers = new Map void>(); + private workerResolvers = new Map void; reject: (err: Error) => void }>(); constructor() { this.audioContext = new (window.AudioContext || (window as typeof window & { webkitAudioContext?: typeof AudioContext }).webkitAudioContext)(); this.worker = new WorkerConstructor(); + this.worker.addEventListener( "message", - (event: MessageEvent<{ requestId: number; peaks: Float32Array }>) => { - const { requestId, peaks } = event.data; - const resolve = this.workerResolvers.get(requestId); - if (!resolve) return; + (event: MessageEvent<{ requestId: number; peaks?: Float32Array; error?: string }>) => { + const { requestId, peaks, error } = event.data; + const resolver = this.workerResolvers.get(requestId); + if (!resolver) return; + this.workerResolvers.delete(requestId); - resolve(peaks); + if (error) { + resolver.reject(new Error(error)); + } else if (peaks) { + resolver.resolve(peaks); + } }, ); + + this.worker.addEventListener("error", (error: ErrorEvent) => { + console.error("[WaveformGenerator] Worker fatal error:", error); + const fatalError = error.error ?? new Error(error.message || "Worker crashed"); + + // Reject all pending requests if the worker itself crashes + for (const resolver of this.workerResolvers.values()) { + resolver.reject(fatalError); + } + this.workerResolvers.clear(); + }); } - private computePeaksWithWorker(channelData: Float32Array, samples: number): Promise { + private computePeaksWithWorker(channels: Float32Array[], samples: number): Promise { return new Promise((resolve, reject) => { const requestId = ++this.workerRequestSeq; - const onError = (error: ErrorEvent) => { - this.worker.removeEventListener("error", onError); - this.workerResolvers.delete(requestId); - reject(error.error ?? new Error(error.message)); - }; - this.worker.addEventListener("error", onError, { once: true }); - this.workerResolvers.set(requestId, (peaks) => { - this.worker.removeEventListener("error", onError); - resolve(peaks); - }); + this.workerResolvers.set(requestId, { resolve, reject }); + this.worker.postMessage( { requestId, - channelData, + channels, samples, }, - [channelData.buffer], + channels.map(c => c.buffer), ); }); } @@ -65,8 +74,14 @@ export class WaveformGenerator { const arrayBuffer = await response.arrayBuffer(); const decoded = await this.audioContext.decodeAudioData(arrayBuffer); - const channelData = decoded.getChannelData(0).slice(); - const peaks = await this.computePeaksWithWorker(channelData, peakCount); + + const channels: Float32Array[] = []; + for (let i = 0; i < decoded.numberOfChannels; i++) { + // We slice to transfer the underlying buffer to the worker + channels.push(decoded.getChannelData(i).slice()); + } + + const peaks = await this.computePeaksWithWorker(channels, peakCount); let max = 0; for (let i = 0; i < peaks.length; i++) { diff --git a/src/components/video-editor/audio/waveform/waveform.worker.ts b/src/components/video-editor/audio/waveform/waveform.worker.ts index 1db7b382..f5332b64 100644 --- a/src/components/video-editor/audio/waveform/waveform.worker.ts +++ b/src/components/video-editor/audio/waveform/waveform.worker.ts @@ -1,34 +1,39 @@ self.onmessage = (e: MessageEvent) => { - const { requestId, channelData, samples } = e.data as { + const { requestId, channels, samples } = e.data as { requestId: number; - channelData: Float32Array; + channels: Float32Array[]; samples: number; }; - if (!channelData || samples <= 0) { + if (!channels || channels.length === 0 || samples <= 0) { const empty = new Float32Array(0); (self as any).postMessage({ requestId, peaks: empty }, [empty.buffer]); return; } try { - const step = Math.max(1, Math.floor(channelData.length / samples)); + const firstChannel = channels[0]; + const step = Math.max(1, Math.floor(firstChannel.length / samples)); const result = new Float32Array(samples); for (let i = 0; i < samples; i++) { const start = i * step; - const end = Math.min(start + step, channelData.length); + const end = Math.min(start + step, firstChannel.length); let max = 0; for (let j = start; j < end; j++) { - const val = Math.abs(channelData[j]); - if (val > max) max = val; + for (let c = 0; c < channels.length; c++) { + const val = Math.abs(channels[c][j]); + if (val > max) max = val; + } } result[i] = max; } (self as any).postMessage({ requestId, peaks: result }, [result.buffer]); - } catch { - const empty = new Float32Array(0); - (self as any).postMessage({ requestId, peaks: empty }, [empty.buffer]); + } catch (err) { + (self as any).postMessage({ + requestId, + error: err instanceof Error ? err.message : "Unknown worker error", + }); } }; From d25aaa0901c515f7a1c28c60faac20e8c8952141 Mon Sep 17 00:00:00 2001 From: Alan Trebugeais Date: Sat, 9 May 2026 21:10:16 +0200 Subject: [PATCH 23/25] audio encoder fix --- src/lib/exporter/audioEncoder.ts | 48 ++++++++++++++++++++++++++++++-- 1 file changed, 46 insertions(+), 2 deletions(-) diff --git a/src/lib/exporter/audioEncoder.ts b/src/lib/exporter/audioEncoder.ts index 72015b5f..8e98091e 100644 --- a/src/lib/exporter/audioEncoder.ts +++ b/src/lib/exporter/audioEncoder.ts @@ -8,7 +8,7 @@ import type { } from "@/components/video-editor/types"; import { buildResolvedAudioPlan, - getSourceTrackIdFromPath, + SourceTrackId, } from "@/lib/exporter/audioRoutingEngine"; import { estimateCompanionAudioStartDelaySeconds } from "@/lib/mediaTiming"; import { resolveMediaElementSource } from "./localMediaSource"; @@ -37,6 +37,48 @@ function resolveSourceTrackGain( return Math.max(0, Math.min(2, settings.volume * normalizeGain)); } +export function getSourceTrackIdFromPath(audioPath: string): SourceTrackId { + const normalized = audioPath.toLowerCase(); + // Check for common patterns like .mic., -mic., mic.mp4, etc. + if ( + normalized.includes(".mic.") || + normalized.includes("-mic.") || + normalized.includes("_mic_") || + normalized.includes("/mic.") || + normalized.includes("\\mic.") || + normalized.endsWith("mic.mp4") || + normalized.endsWith("mic.m4a") || + normalized.endsWith("mic.wav") + ) { + return "mic"; + } + if ( + normalized.includes(".system.") || + normalized.includes("-system.") || + normalized.includes("_system_") || + normalized.includes("/system.") || + normalized.includes("\\system.") || + normalized.endsWith("system.mp4") || + normalized.endsWith("system.m4a") || + normalized.endsWith("system.wav") + ) { + return "system"; + } + return "mixed"; +} + +export function hasNonDefaultSourceTrackSettings( + sourceAudioTrackSettings?: SourceAudioTrackSettings, +) { + if (!sourceAudioTrackSettings) { + return false; + } + return Object.values(sourceAudioTrackSettings).some( + (settings) => + Math.abs((settings?.volume ?? 1) - 1) > 0.0005 || Boolean(settings?.normalize), + ); +} + interface TimelineSlice { sourceStartMs: number; sourceEndMs: number; @@ -198,7 +240,9 @@ export class AudioProcessor { if ( sortedSpeedRegions.length > 0 || sortedAudioRegions.length > 0 || - needsSourceAudioMixing + needsSourceAudioMixing || + hasNonDefaultSourceTrackSettings(sourceAudioTrackSettings) || + (clipRegions ?? []).some((clip) => Boolean(clip.muted)) ) { await this.renderAndMuxOfflineAudio( videoUrl, From d7a673446403ceed5573b208e932c35944470129 Mon Sep 17 00:00:00 2001 From: Alan Trebugeais Date: Sat, 9 May 2026 21:22:22 +0200 Subject: [PATCH 24/25] fix exporter not taking varaibles --- src/components/video-editor/VideoEditor.tsx | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/components/video-editor/VideoEditor.tsx b/src/components/video-editor/VideoEditor.tsx index f9c75bbe..71a29d75 100644 --- a/src/components/video-editor/VideoEditor.tsx +++ b/src/components/video-editor/VideoEditor.tsx @@ -4352,6 +4352,10 @@ export default function VideoEditor() { encodingMode, useModernNativeStaticLayout: useExperimentalNativeExport, }); + const sourceAudioTrackSettingsForExport = + selectedClipId !== null + ? audio.selectedClipSourceAudioTrackSettings + : audio.activeSourceAudioTrackSettings; const exporterConfig = { videoUrl: videoPath, @@ -4421,7 +4425,7 @@ export default function VideoEditor() { sourceAudioFallbackPaths: audio.sourceAudioFallbackPaths, sourceAudioFallbackStartDelayMsByPath: audio.sourceAudioFallbackStartDelayMsByPath, - sourceAudioTrackSettings: audio.activeSourceAudioTrackSettings, + sourceAudioTrackSettings: sourceAudioTrackSettingsForExport, previewWidth, previewHeight, onProgress: (progress: ExportProgress) => { @@ -4670,6 +4674,8 @@ export default function VideoEditor() { audioRegions, audio.sourceAudioFallbackPaths, audio.sourceAudioFallbackStartDelayMsByPath, + audio.activeSourceAudioTrackSettings, + audio.selectedClipSourceAudioTrackSettings, exportEncodingMode, exportBackendPreference, exportPipelineModel, @@ -4701,6 +4707,7 @@ export default function VideoEditor() { smokeExportConfig.shadowIntensity, effectiveSpeedRegions, frame, + selectedClipId, smokeExportConfig.encodingMode, smokeExportConfig.fps, smokeExportConfig.quality, From a18192c6df60f273b083dc89c38cd8e22225ab37 Mon Sep 17 00:00:00 2001 From: Alan Trebugeais Date: Sat, 9 May 2026 21:37:55 +0200 Subject: [PATCH 25/25] fix valid code rabbits comments --- .../audio/useSourceAudioTrackSettings.ts | 92 +++++++++++-------- .../audio/waveform/waveform.worker.ts | 33 ++++--- .../components/viewport/TimelineCanvas.tsx | 2 +- .../components/waveform/AudioWaveform.tsx | 5 +- .../components/wrapper/TimelineWrapper.tsx | 2 +- .../timeline/core/timelineTypes.ts | 1 + .../timeline/model/timelineModel.ts | 25 +++-- src/i18n/locales/fr/settings.json | 6 +- 8 files changed, 99 insertions(+), 67 deletions(-) diff --git a/src/components/video-editor/audio/useSourceAudioTrackSettings.ts b/src/components/video-editor/audio/useSourceAudioTrackSettings.ts index 799d6db9..22688534 100644 --- a/src/components/video-editor/audio/useSourceAudioTrackSettings.ts +++ b/src/components/video-editor/audio/useSourceAudioTrackSettings.ts @@ -110,47 +110,59 @@ export function useSourceAudioTrackSettings({ [defaultSourceAudioTrackSettings, sourceAudioTrackSettingsByClip], ); - const onSelectedClipSourceAudioTrackVolumeChange = useCallback( - (id: string, volume: number) => { - if (!selectedClipId) return; - setSourceAudioTrackSettingsByClip((prev) => { - const prevClip = prev[selectedClipId] ?? defaultSourceAudioTrackSettings; - return { - ...prev, - [selectedClipId]: { - ...prevClip, - [id]: { - volume: Number.isFinite(volume) - ? Math.max(0, Math.min(2, volume)) - : (prevClip[id]?.volume ?? 1), - normalize: prevClip[id]?.normalize ?? false, - }, - }, - }; - }); - }, - [defaultSourceAudioTrackSettings, selectedClipId], - ); + const onSelectedClipSourceAudioTrackVolumeChange = useCallback( + (id: string, volume: number) => { + if (!selectedClipId) return; + setSourceAudioTrackSettingsByClip((prev) => { + const prevClip = prev[selectedClipId] ?? defaultSourceAudioTrackSettings; + const nextVolume = Number.isFinite(volume) + ? Math.max(0, Math.min(2, volume)) + : (prevClip[id]?.volume ?? 1); + const prevNormalize = prevClip[id]?.normalize ?? false; + if ( + prevClip[id]?.volume === nextVolume && + prevClip[id]?.normalize === prevNormalize + ) { + return prev; + } + return { + ...prev, + [selectedClipId]: { + ...prevClip, + [id]: { + volume: nextVolume, + normalize: prevNormalize, + }, + }, + }; + }); + }, + [defaultSourceAudioTrackSettings, selectedClipId], + ); - const onSelectedClipSourceAudioTrackNormalizeChange = useCallback( - (id: string, normalize: boolean) => { - if (!selectedClipId) return; - setSourceAudioTrackSettingsByClip((prev) => { - const prevClip = prev[selectedClipId] ?? defaultSourceAudioTrackSettings; - return { - ...prev, - [selectedClipId]: { - ...prevClip, - [id]: { - volume: prevClip[id]?.volume ?? 1, - normalize, - }, - }, - }; - }); - }, - [defaultSourceAudioTrackSettings, selectedClipId], - ); + const onSelectedClipSourceAudioTrackNormalizeChange = useCallback( + (id: string, normalize: boolean) => { + if (!selectedClipId) return; + setSourceAudioTrackSettingsByClip((prev) => { + const prevClip = prev[selectedClipId] ?? defaultSourceAudioTrackSettings; + const prevVolume = prevClip[id]?.volume ?? 1; + if (prevClip[id]?.normalize === normalize) { + return prev; + } + return { + ...prev, + [selectedClipId]: { + ...prevClip, + [id]: { + volume: prevVolume, + normalize, + }, + }, + }; + }); + }, + [defaultSourceAudioTrackSettings, selectedClipId], + ); return { sourceAudioTrackMeta, diff --git a/src/components/video-editor/audio/waveform/waveform.worker.ts b/src/components/video-editor/audio/waveform/waveform.worker.ts index f5332b64..1e43df98 100644 --- a/src/components/video-editor/audio/waveform/waveform.worker.ts +++ b/src/components/video-editor/audio/waveform/waveform.worker.ts @@ -1,24 +1,33 @@ -self.onmessage = (e: MessageEvent) => { - const { requestId, channels, samples } = e.data as { - requestId: number; - channels: Float32Array[]; - samples: number; - }; +type WaveformWorkerRequest = { + requestId: number; + channels: Float32Array[]; + samples: number; +}; + +interface WorkerContext { + onmessage: (e: MessageEvent) => void; + postMessage: (message: any, transfer?: Transferable[]) => void; +} + +const workerScope = self as unknown as WorkerContext; + +workerScope.onmessage = (e: MessageEvent) => { + const { requestId, channels, samples } = e.data; if (!channels || channels.length === 0 || samples <= 0) { const empty = new Float32Array(0); - (self as any).postMessage({ requestId, peaks: empty }, [empty.buffer]); + workerScope.postMessage({ requestId, peaks: empty }, [empty.buffer]); return; } try { const firstChannel = channels[0]; - const step = Math.max(1, Math.floor(firstChannel.length / samples)); const result = new Float32Array(samples); + const total = firstChannel.length; for (let i = 0; i < samples; i++) { - const start = i * step; - const end = Math.min(start + step, firstChannel.length); + const start = Math.floor((i * total) / samples); + const end = Math.floor(((i + 1) * total) / samples); let max = 0; for (let j = start; j < end; j++) { for (let c = 0; c < channels.length; c++) { @@ -29,9 +38,9 @@ self.onmessage = (e: MessageEvent) => { result[i] = max; } - (self as any).postMessage({ requestId, peaks: result }, [result.buffer]); + workerScope.postMessage({ requestId, peaks: result }, [result.buffer]); } catch (err) { - (self as any).postMessage({ + workerScope.postMessage({ requestId, error: err instanceof Error ? err.message : "Unknown worker error", }); diff --git a/src/components/video-editor/timeline/components/viewport/TimelineCanvas.tsx b/src/components/video-editor/timeline/components/viewport/TimelineCanvas.tsx index 0c742069..aab23f76 100644 --- a/src/components/video-editor/timeline/components/viewport/TimelineCanvas.tsx +++ b/src/components/video-editor/timeline/components/viewport/TimelineCanvas.tsx @@ -399,7 +399,7 @@ const TimelineCanvasRows = memo(function TimelineCanvasRows({ onSelect={() => onSelectClip?.(item.id)} variant="audio" waveformPeaks={track.peaks} - waveformSegmentSpan={liveSpanPreviewById?.[item.id] ?? item.span} + waveformSegmentSpan={item.sourceSpan ?? item.span} waveformGain={Math.max(0, Math.min(2, settings.volume))} waveformNormalize={Boolean(settings.normalize)} muted={item.muted} diff --git a/src/components/video-editor/timeline/components/waveform/AudioWaveform.tsx b/src/components/video-editor/timeline/components/waveform/AudioWaveform.tsx index ae06fd49..2380d598 100644 --- a/src/components/video-editor/timeline/components/waveform/AudioWaveform.tsx +++ b/src/components/video-editor/timeline/components/waveform/AudioWaveform.tsx @@ -51,7 +51,10 @@ function AudioWaveformComponent({ const draw = () => { const now = performance.now(); - if (now - lastDrawAtRef.current < 33) return; + if (now - lastDrawAtRef.current < 33) { + rafId = requestAnimationFrame(draw); + return; + } lastDrawAtRef.current = now; const ctx = canvas.getContext("2d"); diff --git a/src/components/video-editor/timeline/components/wrapper/TimelineWrapper.tsx b/src/components/video-editor/timeline/components/wrapper/TimelineWrapper.tsx index db0925c1..f09b4001 100644 --- a/src/components/video-editor/timeline/components/wrapper/TimelineWrapper.tsx +++ b/src/components/video-editor/timeline/components/wrapper/TimelineWrapper.tsx @@ -146,7 +146,7 @@ export default function TimelineWrapper({ ? (event.activatorEvent as PointerEvent).clientX + (event.delta?.x ?? 0) : undefined; if (span) showTooltip(span, screenX); - const moved = Math.abs(event.delta?.x ?? 0) > 0.01; + const moved = Math.hypot(event.delta?.x ?? 0, event.delta?.y ?? 0) > 0.01; if (moved) { onLiveSpanPreviewChange?.(event.active.id as string, span ?? null); } diff --git a/src/components/video-editor/timeline/core/timelineTypes.ts b/src/components/video-editor/timeline/core/timelineTypes.ts index 573daff4..d1f705d7 100644 --- a/src/components/video-editor/timeline/core/timelineTypes.ts +++ b/src/components/video-editor/timeline/core/timelineTypes.ts @@ -31,6 +31,7 @@ export interface TimelineRenderItem { id: string; rowId: string; span: Span; + sourceSpan?: Span; label: string; audioPath?: string; audioGain?: number; diff --git a/src/components/video-editor/timeline/model/timelineModel.ts b/src/components/video-editor/timeline/model/timelineModel.ts index 70f63c4c..96681351 100644 --- a/src/components/video-editor/timeline/model/timelineModel.ts +++ b/src/components/video-editor/timeline/model/timelineModel.ts @@ -47,15 +47,22 @@ export function buildTimelineItems(params: { variant: "zoom", })); - const clips: TimelineRenderItem[] = clipRegions.map((region, index) => ({ - id: region.id, - rowId: CLIP_ROW_ID, - span: { start: region.startMs, end: region.endMs }, - label: `Clip ${index + 1}`, - showSourceAudio: region.showSourceAudio, - muted: Boolean(region.muted), - variant: "clip", - })); + const clips: TimelineRenderItem[] = clipRegions.map((region, index) => { + const displayDurationMs = Math.max(0, region.endMs - region.startMs); + const speed = Number.isFinite(region.speed) && region.speed > 0 ? region.speed : 1; + const sourceEndMs = region.startMs + displayDurationMs * speed; + + return { + id: region.id, + rowId: CLIP_ROW_ID, + span: { start: region.startMs, end: region.endMs }, + sourceSpan: { start: region.startMs, end: sourceEndMs }, + label: `Clip ${index + 1}`, + showSourceAudio: region.showSourceAudio, + muted: Boolean(region.muted), + variant: "clip", + }; + }); const annotations: TimelineRenderItem[] = annotationRegions.map((region) => ({ id: region.id, diff --git a/src/i18n/locales/fr/settings.json b/src/i18n/locales/fr/settings.json index 4f790b97..200ecaa0 100644 --- a/src/i18n/locales/fr/settings.json +++ b/src/i18n/locales/fr/settings.json @@ -190,10 +190,10 @@ "volumeTitle": "Audio", "volume": "Volume", "normalize": "Normaliser", - "sourceTracksTitle": "Audio source du clip", - "systemLabel": "Son Système", + "sourceTracksTitle": "Source audio du clip", + "systemLabel": "Son système", "micLabel": "Microphone", "mixedLabel": "Source", - "deleteRegion": "Supprimer l'audio" + "deleteRegion": "Supprimer la zone audio" } }