diff --git a/electron/ipc/recording/mac.ts b/electron/ipc/recording/mac.ts index 09dd5aa7..01951345 100644 --- a/electron/ipc/recording/mac.ts +++ b/electron/ipc/recording/mac.ts @@ -30,9 +30,9 @@ import { validateRecordedVideo, } from "./diagnostics"; import { emitRecordingInterrupted } from "./events"; +import { getFinalMacCompanionAudioPath } from "./macCompanionAudio"; import { pruneAutoRecordings } from "./prune"; - export function waitForNativeCaptureStart(process: ChildProcessWithoutNullStreams) { return new Promise((resolve, reject) => { const timer = setTimeout(() => { @@ -119,13 +119,11 @@ export async function muxNativeMacRecordingWithAudio( microphonePath?: string | null, ) { console.log("[mac-mux] Optimization active: keeping tracks separate."); - - const videoPathWithoutExt = videoPath.replace(/\.[^.]+$/u, ""); // Optimization: instead of heavy FFmpeg muxing, we ensure audio sidecars // are available alongside the video for the editor. if (systemAudioPath) { - const finalSystemPath = `${videoPathWithoutExt}.system.wav`; + const finalSystemPath = getFinalMacCompanionAudioPath(videoPath, systemAudioPath, "system"); try { const stat = await fs.stat(systemAudioPath); if (stat.size > 0 && systemAudioPath !== finalSystemPath) { @@ -137,7 +135,7 @@ export async function muxNativeMacRecordingWithAudio( } if (microphonePath) { - const finalMicPath = `${videoPathWithoutExt}.mic.wav`; + const finalMicPath = getFinalMacCompanionAudioPath(videoPath, microphonePath, "mic"); try { const stat = await fs.stat(microphonePath); if (stat.size > 0 && microphonePath !== finalMicPath) { diff --git a/electron/ipc/recording/macCompanionAudio.test.ts b/electron/ipc/recording/macCompanionAudio.test.ts new file mode 100644 index 00000000..269a7662 --- /dev/null +++ b/electron/ipc/recording/macCompanionAudio.test.ts @@ -0,0 +1,35 @@ +import { describe, expect, it } from "vitest"; + +import { getFinalMacCompanionAudioPath } from "./macCompanionAudio"; + +describe("mac companion audio paths", () => { + it("preserves the helper's AAC container extension", () => { + expect( + getFinalMacCompanionAudioPath( + "/Users/egg/Recordly/recording-1.mp4", + "/Users/egg/Recordly/recording-1.mic.m4a", + "mic", + ), + ).toBe("/Users/egg/Recordly/recording-1.mic.m4a"); + }); + + it("preserves legacy sidecar extensions instead of renaming bytes", () => { + expect( + getFinalMacCompanionAudioPath( + "/Users/egg/Recordly/recording-1.mp4", + "/tmp/recordly-native.system.webm", + "system", + ), + ).toBe("/Users/egg/Recordly/recording-1.system.webm"); + }); + + it("keeps dotted directories when the video path has no extension", () => { + expect( + getFinalMacCompanionAudioPath( + "/Users/egg/Recordly.videos/recording-1", + "/tmp/recordly-native.mic.m4a", + "mic", + ), + ).toBe("/Users/egg/Recordly.videos/recording-1.mic.m4a"); + }); +}); diff --git a/electron/ipc/recording/macCompanionAudio.ts b/electron/ipc/recording/macCompanionAudio.ts new file mode 100644 index 00000000..fdfc9e26 --- /dev/null +++ b/electron/ipc/recording/macCompanionAudio.ts @@ -0,0 +1,16 @@ +import path from "node:path"; + +export type MacCompanionAudioSuffix = "system" | "mic"; + +export function getFinalMacCompanionAudioPath( + videoPath: string, + sourceAudioPath: string, + suffix: MacCompanionAudioSuffix, +) { + const separatorIndex = Math.max(videoPath.lastIndexOf("/"), videoPath.lastIndexOf("\\")); + const videoDirectory = separatorIndex >= 0 ? videoPath.slice(0, separatorIndex + 1) : ""; + const videoFileName = separatorIndex >= 0 ? videoPath.slice(separatorIndex + 1) : videoPath; + const videoPathWithoutExt = `${videoDirectory}${path.parse(videoFileName).name}`; + const sourceExtension = path.extname(sourceAudioPath).toLowerCase() || ".m4a"; + return `${videoPathWithoutExt}.${suffix}${sourceExtension}`; +} diff --git a/src/components/video-editor/timeline/TimelineEditor.tsx b/src/components/video-editor/timeline/TimelineEditor.tsx index 02b8b9b7..041bc318 100644 --- a/src/components/video-editor/timeline/TimelineEditor.tsx +++ b/src/components/video-editor/timeline/TimelineEditor.tsx @@ -1,20 +1,13 @@ -import type { Span } from "dnd-timeline"; import { Plus } from "@phosphor-icons/react"; -import { - forwardRef, - useEffect, - useMemo, - useRef, - useState, -} from "react"; -import { useScopedT } from "@/contexts/I18nContext"; -import { useShortcuts } from "@/contexts/ShortcutsContext"; -import { fromFileUrl } from "../projectPersistence"; +import type { Span } from "dnd-timeline"; +import { forwardRef, useEffect, useMemo, useRef, useState } from "react"; import type { SourceAudioTrackMeta, SourceAudioTrackSettings, - SourceAudioTrackWithPeaks, } from "@/components/video-editor/audio/audioTypes"; +import { useScopedT } from "@/contexts/I18nContext"; +import { useShortcuts } from "@/contexts/ShortcutsContext"; +import { fromFileUrl } from "../projectPersistence"; import type { AnnotationRegion, AudioRegion, @@ -26,12 +19,16 @@ import type { ZoomRegion, } from "../types"; import KeyframeMarkers from "./components/markers/KeyframeMarkers"; +import TimelineCanvas from "./components/viewport/TimelineCanvas"; import TimelineWrapper from "./components/wrapper/TimelineWrapper"; -import { useTimelineAudioPeaks } from "./hooks/useTimelineAudioPeaks"; import { calculateTimelineScale } from "./core/time"; +import { useTimelineAudioPeaks } from "./hooks/useTimelineAudioPeaks"; import { useTimelineEditorRuntime } from "./hooks/useTimelineEditorRuntime"; import { useTimelineRange } from "./hooks/useTimelineRange"; -import TimelineCanvas from "./components/viewport/TimelineCanvas"; +import { + buildSourceSidecarPathCandidates, + buildTimelineSourceAudioTracks, +} from "./sourceAudioTracks"; export interface TimelineEditorProps { videoDuration: number; @@ -77,9 +74,7 @@ export interface TimelineEditorProps { showSourceAudioTrack?: boolean; onSourceAudioAvailabilityChange?: (available: boolean) => void; sourceAudioTrackSettings?: SourceAudioTrackSettings; - getSourceAudioTrackSettingsForClip?: ( - clipId: string | null, - ) => SourceAudioTrackSettings; + getSourceAudioTrackSettingsForClip?: (clipId: string | null) => SourceAudioTrackSettings; onSourceAudioTracksMetaChange?: (tracks: SourceAudioTrackMeta) => void; } @@ -98,16 +93,6 @@ function extractLocalPathFromMediaServerUrl(input: string | null | undefined): s } } -function buildSourceSidecarPath(source: string, suffix: "mic" | "system"): string { - const normalized = source.replace(/\\/g, "/"); - const lastSlash = normalized.lastIndexOf("/"); - const dir = lastSlash >= 0 ? normalized.slice(0, lastSlash + 1) : ""; - const fileName = lastSlash >= 0 ? normalized.slice(lastSlash + 1) : normalized; - const dotIndex = fileName.lastIndexOf("."); - const baseName = dotIndex > 0 ? fileName.slice(0, dotIndex) : fileName; - return `${dir}${baseName}.${suffix}.wav`; -} - export interface TimelineEditorHandle { addZoom: () => void; suggestZooms: () => void; @@ -117,7 +102,6 @@ export interface TimelineEditorHandle { keyframes: { id: string; time: number }[]; } - const TimelineEditor = forwardRef( function TimelineEditor( { @@ -209,9 +193,7 @@ const TimelineEditor = forwardRef( ...(newStart > oldClip.startMs ? [{ startMs: oldClip.startMs, endMs: newStart }] : []), - ...(newEnd < oldClip.endMs - ? [{ startMs: newEnd, endMs: oldClip.endMs }] - : []), + ...(newEnd < oldClip.endMs ? [{ startMs: newEnd, endMs: oldClip.endMs }] : []), ]; const startDelta = newStart - oldClip.startMs; @@ -245,9 +227,8 @@ const TimelineEditor = forwardRef( return { previewSpans, hiddenZoomIds }; }, [clipRegions, liveSpanPreviewById, zoomRegions]); const { shortcuts: keyShortcuts, isMac } = useShortcuts(); - const { peaks: sourceAudioPeaks, loading: sourceAudioLoading } = useTimelineAudioPeaks(videoPath, { - enableSourceSidecarFallback: true, - }); + const { peaks: sourceAudioPeaks, loading: sourceAudioLoading } = + useTimelineAudioPeaks(videoPath); const localSourcePath = useMemo(() => { if (!videoPath) return null; return ( @@ -255,56 +236,67 @@ const TimelineEditor = forwardRef( (/^file:\/\//i.test(videoPath) ? fromFileUrl(videoPath) : videoPath) ); }, [videoPath]); - const micSidecarPath = useMemo( - () => (localSourcePath ? buildSourceSidecarPath(localSourcePath, "mic") : null), + const micSidecarPaths = useMemo( + () => (localSourcePath ? buildSourceSidecarPathCandidates(localSourcePath, "mic") : []), [localSourcePath], ); - const systemSidecarPath = useMemo( - () => (localSourcePath ? buildSourceSidecarPath(localSourcePath, "system") : null), + const micSidecarFallbackPaths = useMemo(() => micSidecarPaths.slice(1), [micSidecarPaths]); + const systemSidecarPaths = useMemo( + () => + localSourcePath ? buildSourceSidecarPathCandidates(localSourcePath, "system") : [], [localSourcePath], ); - const { peaks: micSidecarPeaks, loading: micSidecarLoading } = useTimelineAudioPeaks(micSidecarPath); - const { peaks: systemSidecarPeaks, loading: systemSidecarLoading } = useTimelineAudioPeaks(systemSidecarPath); - const sourceAudioTracks = useMemo(() => { - if (systemSidecarPeaks || micSidecarPeaks) { - const tracks: SourceAudioTrackWithPeaks[] = []; - if (systemSidecarPeaks) - tracks.push({ - id: "system", - label: t("audio.systemLabel", "Source System"), - peaks: systemSidecarPeaks, - }); - if (micSidecarPeaks) - tracks.push({ - id: "mic", - label: t("audio.micLabel", "Source Mic"), - peaks: micSidecarPeaks, - }); - return tracks; - } - return sourceAudioPeaks - ? [ - { - id: "mixed", - label: t("audio.mixedLabel", "Source"), - peaks: sourceAudioPeaks, - }, - ] - : []; - }, [micSidecarPeaks, sourceAudioPeaks, systemSidecarPeaks, t]); + const systemSidecarFallbackPaths = useMemo( + () => systemSidecarPaths.slice(1), + [systemSidecarPaths], + ); + const { peaks: micSidecarPeaks, loading: micSidecarLoading } = useTimelineAudioPeaks( + micSidecarPaths[0] ?? null, + { fallbackResources: micSidecarFallbackPaths }, + ); + const { peaks: systemSidecarPeaks, loading: systemSidecarLoading } = useTimelineAudioPeaks( + systemSidecarPaths[0] ?? null, + { + fallbackResources: systemSidecarFallbackPaths, + }, + ); + const sourceAudioTracks = useMemo( + () => + buildTimelineSourceAudioTracks({ + sourceAudioPeaks, + micSidecarPeaks, + systemSidecarPeaks, + labels: { + system: t("audio.systemLabel", "Source System"), + mic: t("audio.micLabel", "Source Mic"), + mixed: t("audio.mixedLabel", "Source"), + }, + }), + [micSidecarPeaks, sourceAudioPeaks, systemSidecarPeaks, t], + ); const isLoading = useMemo(() => { // If we are still actively trying to load audio peaks (main or sidecars) - if (videoPath && (sourceAudioLoading || micSidecarLoading || systemSidecarLoading)) return true; + if (videoPath && (sourceAudioLoading || micSidecarLoading || systemSidecarLoading)) + return true; // Robust telemetry loading detection: // If a source path is set but telemetry hasn't arrived (or failed/retried) for it yet. if (videoSourcePath && cursorTelemetrySourcePath !== videoSourcePath) return true; return false; - }, [videoPath, videoSourcePath, cursorTelemetrySourcePath, sourceAudioLoading, micSidecarLoading, systemSidecarLoading]); + }, [ + videoPath, + videoSourcePath, + cursorTelemetrySourcePath, + sourceAudioLoading, + micSidecarLoading, + systemSidecarLoading, + ]); useEffect(() => { - onSourceAudioTracksMetaChange?.(sourceAudioTracks.map((t) => ({ id: t.id, label: t.label }))); + onSourceAudioTracksMetaChange?.( + sourceAudioTracks.map((t) => ({ id: t.id, label: t.label })), + ); }, [onSourceAudioTracksMetaChange, sourceAudioTracks]); void sourceAudioTrackSettings; useEffect(() => { diff --git a/src/components/video-editor/timeline/hooks/useTimelineAudioPeaks.ts b/src/components/video-editor/timeline/hooks/useTimelineAudioPeaks.ts index 87ebc357..cf5ca393 100644 --- a/src/components/video-editor/timeline/hooks/useTimelineAudioPeaks.ts +++ b/src/components/video-editor/timeline/hooks/useTimelineAudioPeaks.ts @@ -1,10 +1,12 @@ import { useEffect, useRef, useState } from "react"; import { resolveMediaResourceUrl } from "@/lib/exporter/localMediaSource"; -import { fromFileUrl } from "../../projectPersistence"; import { waveformGenerator } from "../../audio/waveform/WaveformGenerator"; +import { fromFileUrl } from "../../projectPersistence"; import { WAVEFORM_DEFAULT_PEAK_COUNT } from "../core/constants"; import type { AudioPeaksData } from "../core/timelineTypes"; +const EMPTY_FALLBACK_RESOURCES: string[] = []; + function buildSidecarAudioCandidates(sourcePath: string): string[] { const normalized = sourcePath.replace(/\\/g, "/"); const lastSlash = normalized.lastIndexOf("/"); @@ -18,6 +20,8 @@ function buildSidecarAudioCandidates(sourcePath: string): string[] { `${dir}${baseName}.mic.wav`, `${dir}${baseName}.system.m4a`, `${dir}${baseName}.mic.m4a`, + `${dir}${baseName}.system.webm`, + `${dir}${baseName}.mic.webm`, ]; } @@ -37,6 +41,7 @@ function extractLocalPathFromMediaServerUrl(input: string): string | null { interface TimelineAudioPeaksOptions { enableSourceSidecarFallback?: boolean; + fallbackResources?: string[]; peakCount?: number; } @@ -53,6 +58,7 @@ export function useTimelineAudioPeaks( const [loading, setLoading] = useState(false); const sourceRef = useRef(mediaResource); const enableSourceSidecarFallback = options.enableSourceSidecarFallback ?? false; + const fallbackResources = options.fallbackResources ?? EMPTY_FALLBACK_RESOURCES; const peakCount = options.peakCount ?? WAVEFORM_DEFAULT_PEAK_COUNT; useEffect(() => { @@ -83,25 +89,29 @@ export function useTimelineAudioPeaks( // fallthrough } - if (!enableSourceSidecarFallback) { + if (!enableSourceSidecarFallback && fallbackResources.length === 0) { if (!cancelled && sourceRef.current === mediaResource) { setLoading(false); } return; } - const localPathFromServer = extractLocalPathFromMediaServerUrl(mediaResource); - const localSourcePath = - localPathFromServer || - (/^file:\/\//i.test(mediaResource) ? fromFileUrl(mediaResource) : mediaResource); - if (!localSourcePath) { - if (!cancelled && sourceRef.current === mediaResource) { - setLoading(false); + let sourceSidecarCandidates: string[] = []; + if (enableSourceSidecarFallback) { + const localPathFromServer = extractLocalPathFromMediaServerUrl(mediaResource); + const localSourcePath = + localPathFromServer || + (/^file:\/\//i.test(mediaResource) + ? fromFileUrl(mediaResource) + : mediaResource); + if (localSourcePath) { + sourceSidecarCandidates = buildSidecarAudioCandidates(localSourcePath); } - return; } - const candidates = buildSidecarAudioCandidates(localSourcePath); + const candidates = Array.from( + new Set([...fallbackResources, ...sourceSidecarCandidates]), + ); for (const candidate of candidates) { try { const result = await tryGenerate(candidate); @@ -125,7 +135,7 @@ export function useTimelineAudioPeaks( return () => { cancelled = true; }; - }, [mediaResource, enableSourceSidecarFallback, peakCount]); + }, [mediaResource, enableSourceSidecarFallback, fallbackResources, peakCount]); return { peaks, loading }; } diff --git a/src/components/video-editor/timeline/sourceAudioTracks.test.ts b/src/components/video-editor/timeline/sourceAudioTracks.test.ts new file mode 100644 index 00000000..f7c809e6 --- /dev/null +++ b/src/components/video-editor/timeline/sourceAudioTracks.test.ts @@ -0,0 +1,90 @@ +import { describe, expect, it } from "vitest"; +import type { AudioPeaksData } from "./core/timelineTypes"; +import { + buildSourceSidecarPathCandidates, + buildTimelineSourceAudioTracks, +} from "./sourceAudioTracks"; + +function peaks(id: number): AudioPeaksData { + return { + durationMs: 1000, + peaks: new Float32Array([id]), + }; +} + +const labels = { + system: "Source System", + mic: "Source Mic", + mixed: "Source", +}; + +describe("timeline source audio tracks", () => { + it("builds candidates for Windows and macOS sidecar containers", () => { + expect(buildSourceSidecarPathCandidates("C:\\Recordly\\recording-1.mp4", "mic")).toEqual([ + "C:/Recordly/recording-1.mic.wav", + "C:/Recordly/recording-1.mic.m4a", + "C:/Recordly/recording-1.mic.webm", + ]); + }); + + it("keeps embedded system audio controllable when mic is a sidecar", () => { + const source = peaks(1); + const mic = peaks(2); + + expect( + buildTimelineSourceAudioTracks({ + sourceAudioPeaks: source, + micSidecarPeaks: mic, + systemSidecarPeaks: null, + labels, + }), + ).toEqual([ + { id: "system", label: "Source System", peaks: source }, + { id: "mic", label: "Source Mic", peaks: mic }, + ]); + }); + + it("does not invent a system track when only the mic sidecar exists", () => { + const mic = peaks(2); + + expect( + buildTimelineSourceAudioTracks({ + sourceAudioPeaks: null, + micSidecarPeaks: mic, + systemSidecarPeaks: null, + labels, + }), + ).toEqual([{ id: "mic", label: "Source Mic", peaks: mic }]); + }); + + it("uses dedicated sidecars over the embedded track when both source tracks exist", () => { + const source = peaks(1); + const system = peaks(2); + const mic = peaks(3); + + expect( + buildTimelineSourceAudioTracks({ + sourceAudioPeaks: source, + micSidecarPeaks: mic, + systemSidecarPeaks: system, + labels, + }), + ).toEqual([ + { id: "system", label: "Source System", peaks: system }, + { id: "mic", label: "Source Mic", peaks: mic }, + ]); + }); + + it("falls back to one mixed source track when no dedicated sidecar exists", () => { + const source = peaks(1); + + expect( + buildTimelineSourceAudioTracks({ + sourceAudioPeaks: source, + micSidecarPeaks: null, + systemSidecarPeaks: null, + labels, + }), + ).toEqual([{ id: "mixed", label: "Source", peaks: source }]); + }); +}); diff --git a/src/components/video-editor/timeline/sourceAudioTracks.ts b/src/components/video-editor/timeline/sourceAudioTracks.ts new file mode 100644 index 00000000..46a089ee --- /dev/null +++ b/src/components/video-editor/timeline/sourceAudioTracks.ts @@ -0,0 +1,68 @@ +import type { SourceAudioTrackWithPeaks } from "@/components/video-editor/audio/audioTypes"; +import type { AudioPeaksData } from "./core/timelineTypes"; + +const SOURCE_SIDECAR_EXTENSIONS = [".wav", ".m4a", ".webm"] as const; + +export function buildSourceSidecarPathCandidates( + source: string, + suffix: "mic" | "system", +): string[] { + const normalized = source.replace(/\\/g, "/"); + const lastSlash = normalized.lastIndexOf("/"); + const dir = lastSlash >= 0 ? normalized.slice(0, lastSlash + 1) : ""; + const fileName = lastSlash >= 0 ? normalized.slice(lastSlash + 1) : normalized; + const dotIndex = fileName.lastIndexOf("."); + const baseName = dotIndex > 0 ? fileName.slice(0, dotIndex) : fileName; + return SOURCE_SIDECAR_EXTENSIONS.map((extension) => `${dir}${baseName}.${suffix}${extension}`); +} + +export function buildTimelineSourceAudioTracks({ + sourceAudioPeaks, + micSidecarPeaks, + systemSidecarPeaks, + labels, +}: { + sourceAudioPeaks: AudioPeaksData | null; + micSidecarPeaks: AudioPeaksData | null; + systemSidecarPeaks: AudioPeaksData | null; + labels: { + system: string; + mic: string; + mixed: string; + }; +}): SourceAudioTrackWithPeaks[] { + if (systemSidecarPeaks || micSidecarPeaks) { + const tracks: SourceAudioTrackWithPeaks[] = []; + if (systemSidecarPeaks) { + tracks.push({ + id: "system", + label: labels.system, + peaks: systemSidecarPeaks, + }); + } else if (micSidecarPeaks && sourceAudioPeaks) { + tracks.push({ + id: "system", + label: labels.system, + peaks: sourceAudioPeaks, + }); + } + if (micSidecarPeaks) { + tracks.push({ + id: "mic", + label: labels.mic, + peaks: micSidecarPeaks, + }); + } + return tracks; + } + + return sourceAudioPeaks + ? [ + { + id: "mixed", + label: labels.mixed, + peaks: sourceAudioPeaks, + }, + ] + : []; +} diff --git a/src/lib/exporter/audioEncoder.test.ts b/src/lib/exporter/audioEncoder.test.ts index 309ff8fe..6047b548 100644 --- a/src/lib/exporter/audioEncoder.test.ts +++ b/src/lib/exporter/audioEncoder.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it, vi } from "vitest"; -import { AudioProcessor } from "./audioEncoder"; +import { AudioProcessor, softLimitOfflineMixPeaksInPlace } from "./audioEncoder"; type OfflineRenderTestHarness = AudioProcessor & { decodeAudioFromUrl(url: string): Promise; @@ -26,8 +26,32 @@ type OfflineRenderTestHarness = AudioProcessor & { sourceAudioFallbackStartDelayMsByPath: Record | undefined, muxer: unknown, ): Promise; + renderChunked( + prepared: { + mainBufferEntry: null; + companionEntries: []; + regionEntries: []; + mutedSourceOutputRangesSec: []; + slices: []; + outputDurationMs: number; + numChannels: number; + }, + totalOutputSec: number, + onChunk: ( + rendered: AudioBuffer, + outputOffsetSec: number, + chunkIndex: number, + ) => Promise, + ): Promise; }; +function fakeAudioBuffer(channels: Float32Array[]): AudioBuffer { + return { + numberOfChannels: channels.length, + getChannelData: (channel: number) => channels[channel], + } as AudioBuffer; +} + describe("AudioProcessor offline render preparation", () => { it("keeps embedded source audio separate from external companion sidecars", async () => { const processor = new AudioProcessor() as unknown as OfflineRenderTestHarness; @@ -136,4 +160,71 @@ describe("AudioProcessor offline render preparation", () => { expect(loadAudioFileDemuxer).not.toHaveBeenCalled(); expect(renderAndMuxOfflineAudio).toHaveBeenCalled(); }); + + it("soft-limits mixed peaks before encoding or WAV conversion", () => { + const samples = new Float32Array([ + -1.6, + -0.5, + Number.NEGATIVE_INFINITY, + Number.NaN, + 0, + 0.5, + 0.95, + Number.POSITIVE_INFINITY, + 1.6, + ]); + const changed = softLimitOfflineMixPeaksInPlace(fakeAudioBuffer([samples])); + + expect(changed).toBe(true); + expect(samples[0]).toBeGreaterThanOrEqual(-0.986); + expect(samples[1]).toBe(-0.5); + expect(samples[2]).toBe(0); + expect(samples[3]).toBe(0); + expect(samples[5]).toBe(0.5); + expect(samples[6]).toBeLessThan(0.95); + expect(samples[6]).toBeGreaterThan(0.9); + expect(samples[7]).toBe(0); + expect(samples[8]).toBeLessThanOrEqual(0.986); + }); + + it("runs the offline mix limiter for every rendered chunk", async () => { + const processor = new AudioProcessor() as unknown as OfflineRenderTestHarness; + const renderedSamples = new Float32Array([1.4]); + const renderedBuffer = fakeAudioBuffer([renderedSamples]); + const originalOfflineAudioContext = globalThis.OfflineAudioContext; + ( + globalThis as unknown as { OfflineAudioContext: typeof OfflineAudioContext } + ).OfflineAudioContext = class { + constructor() {} + + startRendering() { + return Promise.resolve(renderedBuffer); + } + } as unknown as typeof OfflineAudioContext; + + try { + let observedPeak = Number.POSITIVE_INFINITY; + await processor.renderChunked( + { + mainBufferEntry: null, + companionEntries: [], + regionEntries: [], + mutedSourceOutputRangesSec: [], + slices: [], + outputDurationMs: 100, + numChannels: 1, + }, + 0.1, + async (rendered) => { + observedPeak = rendered.getChannelData(0)[0] ?? 0; + }, + ); + + expect(observedPeak).toBeLessThanOrEqual(0.986); + } finally { + ( + globalThis as unknown as { OfflineAudioContext: typeof OfflineAudioContext } + ).OfflineAudioContext = originalOfflineAudioContext; + } + }); }); diff --git a/src/lib/exporter/audioEncoder.ts b/src/lib/exporter/audioEncoder.ts index fba88b6b..d66e0bea 100644 --- a/src/lib/exporter/audioEncoder.ts +++ b/src/lib/exporter/audioEncoder.ts @@ -1,20 +1,17 @@ import { WebDemuxer } from "web-demuxer"; +import { SOURCE_AUDIO_NORMALIZE_GAIN } from "@/components/video-editor/audio/audioTypes"; import type { AudioRegion, ClipRegion, - SpeedRegion, SourceAudioTrackSettings, + SpeedRegion, TrimRegion, } from "@/components/video-editor/types"; -import { - buildResolvedAudioPlan, - SourceTrackId, -} from "@/lib/exporter/audioRoutingEngine"; +import { buildResolvedAudioPlan, SourceTrackId } from "@/lib/exporter/audioRoutingEngine"; import { estimateCompanionAudioStartDelaySeconds } from "@/lib/mediaTiming"; import { resolveMediaElementSource } from "./localMediaSource"; import type { VideoMuxer } from "./muxer"; import { resolveSourceTrackRoutingPolicy } from "./sourceTrackRoutingPolicy"; -import { SOURCE_AUDIO_NORMALIZE_GAIN } from "@/components/video-editor/audio/audioTypes"; const AUDIO_BITRATE = 128_000; const DECODE_BACKPRESSURE_LIMIT = 20; @@ -24,6 +21,44 @@ const MP4_AUDIO_CODEC = "mp4a.40.2"; const OFFLINE_AUDIO_SAMPLE_RATE = 48_000; const OFFLINE_ENCODE_CHUNK_FRAMES = 1024; const OFFLINE_CHUNK_DURATION_SEC = 30; +const OFFLINE_MIX_SOFT_LIMITER_THRESHOLD = 0.9; +const OFFLINE_MIX_SOFT_LIMITER_CEILING = 0.985; + +function softLimitSample(sample: number): number { + const magnitude = Math.abs(sample); + if (magnitude <= OFFLINE_MIX_SOFT_LIMITER_THRESHOLD) { + return sample; + } + + const sign = sample < 0 ? -1 : 1; + const kneeRange = 1 - OFFLINE_MIX_SOFT_LIMITER_THRESHOLD; + const limitedMagnitude = + OFFLINE_MIX_SOFT_LIMITER_THRESHOLD + + kneeRange * Math.tanh((magnitude - OFFLINE_MIX_SOFT_LIMITER_THRESHOLD) / kneeRange); + return sign * Math.min(OFFLINE_MIX_SOFT_LIMITER_CEILING, limitedMagnitude); +} + +export function softLimitOfflineMixPeaksInPlace(buffer: AudioBuffer): boolean { + let changed = false; + for (let channel = 0; channel < buffer.numberOfChannels; channel += 1) { + const data = buffer.getChannelData(channel); + for (let index = 0; index < data.length; index += 1) { + const sample = data[index]; + if (!Number.isFinite(sample)) { + data[index] = 0; + changed = true; + continue; + } + + const limited = softLimitSample(sample); + if (limited !== sample) { + data[index] = limited; + changed = true; + } + } + } + return changed; +} function resolveSourceTrackGain( sourceAudioTrackSettings: SourceAudioTrackSettings | undefined, @@ -698,8 +733,11 @@ export class AudioProcessor { if (this.cancelled) throw new Error("Export cancelled"); // Decode companion / sidecar audio files - const companionEntries: Array<{ buffer: AudioBuffer; startDelaySec: number; gain: number }> = - []; + const companionEntries: Array<{ + buffer: AudioBuffer; + startDelaySec: number; + gain: number; + }> = []; const refDuration = mainBuffer?.duration ?? (resolvedPlan.playbackPaths.length > 0 ? await this.getMediaDurationSec(videoUrl) : 0); @@ -947,6 +985,7 @@ export class AudioProcessor { const rendered = await offlineCtx.startRendering(); if (this.cancelled) break; + softLimitOfflineMixPeaksInPlace(rendered); await onChunk(rendered, outputOffsetSec, i); @@ -1459,7 +1498,9 @@ export class AudioProcessor { { startSec: localOutputStartSec + chunkOutputStartSec, endSec: - localOutputStartSec + chunkOutputStartSec + effectiveSourceDurationSec / slice.speed, + localOutputStartSec + + chunkOutputStartSec + + effectiveSourceDurationSec / slice.speed, }, ]; for (const mutedRange of mutedOutputRangesSec) { @@ -1491,7 +1532,8 @@ export class AudioProcessor { const sourceOffsetSec = effectiveBufferStartSec + - (audibleRange.startSec - (localOutputStartSec + chunkOutputStartSec)) * slice.speed; + (audibleRange.startSec - (localOutputStartSec + chunkOutputStartSec)) * + slice.speed; const localStartSec = audibleRange.startSec - chunkOutputStartSec; const sourceDurationSec = audibleDurationSec * slice.speed; @@ -1543,7 +1585,9 @@ export class AudioProcessor { if (copyLength > 0) { for (let c = 0; c < channels; c++) { outBuffer.copyToChannel( - originalBuffer.getChannelData(c).subarray(startSample, startSample + copyLength), + originalBuffer + .getChannelData(c) + .subarray(startSample, startSample + copyLength), c, ); } @@ -1559,7 +1603,7 @@ export class AudioProcessor { const workStartIn = Math.max(0, startSample - paddingInSamples); const workEndIn = Math.min(originalBuffer.length, endSample + paddingInSamples); - + const actualPaddingInStart = startSample - workStartIn; // We expect the output offset for the requested start to be roughly: const actualPaddingOutStart = Math.floor(actualPaddingInStart / speed); @@ -1572,8 +1616,12 @@ export class AudioProcessor { const workOutSamples = Math.floor((workEndIn - workStartIn) / speed) + windowSize * 2; const workOutBuffer = ctx.createBuffer(channels, workOutSamples, sampleRate); - const inDataByChannel = Array.from({ length: channels }, (_, c) => originalBuffer.getChannelData(c)); - const workOutDataByChannel = Array.from({ length: channels }, (_, c) => workOutBuffer.getChannelData(c)); + const inDataByChannel = Array.from({ length: channels }, (_, c) => + originalBuffer.getChannelData(c), + ); + const workOutDataByChannel = Array.from({ length: channels }, (_, c) => + workOutBuffer.getChannelData(c), + ); const window = new Float32Array(windowSize); for (let i = 0; i < windowSize; i++) { @@ -1587,7 +1635,8 @@ export class AudioProcessor { for (let i = 0; i < windowSize; i++) { if (inOffset + i < workEndIn && outOffset + i < workOutSamples) { for (let c = 0; c < channels; c++) { - workOutDataByChannel[c][outOffset + i] += inDataByChannel[c][inOffset + i] * window[i]; + workOutDataByChannel[c][outOffset + i] += + inDataByChannel[c][inOffset + i] * window[i]; } } } @@ -1609,7 +1658,9 @@ export class AudioProcessor { for (let i = 0; i < hopOut; i += 4) { if (outOffset + i < workOutSamples && testOffset + i < workEndIn) { for (let c = 0; c < channels; c++) { - corr += workOutDataByChannel[c][outOffset + i] * inDataByChannel[c][testOffset + i]; + corr += + workOutDataByChannel[c][outOffset + i] * + inDataByChannel[c][testOffset + i]; } } } @@ -1624,7 +1675,8 @@ export class AudioProcessor { for (let i = 0; i < windowSize; i++) { if (bestOffset + i < workEndIn && outOffset + i < workOutSamples) { for (let c = 0; c < channels; c++) { - workOutDataByChannel[c][outOffset + i] += inDataByChannel[c][bestOffset + i] * window[i]; + workOutDataByChannel[c][outOffset + i] += + inDataByChannel[c][bestOffset + i] * window[i]; } } }