fix: preserve mac audio sidecar tracks

This commit is contained in:
wiiiii123
2026-05-29 23:30:39 +07:00
parent be8c97d676
commit 80a513b7c4
9 changed files with 457 additions and 105 deletions
+3 -5
View File
@@ -30,9 +30,9 @@ import {
validateRecordedVideo,
} from "./diagnostics";
import { emitRecordingInterrupted } from "./events";
import { getFinalMacCompanionAudioPath } from "./macCompanionAudio";
import { pruneAutoRecordings } from "./prune";
export function waitForNativeCaptureStart(process: ChildProcessWithoutNullStreams) {
return new Promise<void>((resolve, reject) => {
const timer = setTimeout(() => {
@@ -119,13 +119,11 @@ export async function muxNativeMacRecordingWithAudio(
microphonePath?: string | null,
) {
console.log("[mac-mux] Optimization active: keeping tracks separate.");
const videoPathWithoutExt = videoPath.replace(/\.[^.]+$/u, "");
// Optimization: instead of heavy FFmpeg muxing, we ensure audio sidecars
// are available alongside the video for the editor.
if (systemAudioPath) {
const finalSystemPath = `${videoPathWithoutExt}.system.wav`;
const finalSystemPath = getFinalMacCompanionAudioPath(videoPath, systemAudioPath, "system");
try {
const stat = await fs.stat(systemAudioPath);
if (stat.size > 0 && systemAudioPath !== finalSystemPath) {
@@ -137,7 +135,7 @@ export async function muxNativeMacRecordingWithAudio(
}
if (microphonePath) {
const finalMicPath = `${videoPathWithoutExt}.mic.wav`;
const finalMicPath = getFinalMacCompanionAudioPath(videoPath, microphonePath, "mic");
try {
const stat = await fs.stat(microphonePath);
if (stat.size > 0 && microphonePath !== finalMicPath) {
@@ -0,0 +1,35 @@
import { describe, expect, it } from "vitest";
import { getFinalMacCompanionAudioPath } from "./macCompanionAudio";
describe("mac companion audio paths", () => {
it("preserves the helper's AAC container extension", () => {
expect(
getFinalMacCompanionAudioPath(
"/Users/egg/Recordly/recording-1.mp4",
"/Users/egg/Recordly/recording-1.mic.m4a",
"mic",
),
).toBe("/Users/egg/Recordly/recording-1.mic.m4a");
});
it("preserves legacy sidecar extensions instead of renaming bytes", () => {
expect(
getFinalMacCompanionAudioPath(
"/Users/egg/Recordly/recording-1.mp4",
"/tmp/recordly-native.system.webm",
"system",
),
).toBe("/Users/egg/Recordly/recording-1.system.webm");
});
it("keeps dotted directories when the video path has no extension", () => {
expect(
getFinalMacCompanionAudioPath(
"/Users/egg/Recordly.videos/recording-1",
"/tmp/recordly-native.mic.m4a",
"mic",
),
).toBe("/Users/egg/Recordly.videos/recording-1.mic.m4a");
});
});
@@ -0,0 +1,16 @@
import path from "node:path";
export type MacCompanionAudioSuffix = "system" | "mic";
export function getFinalMacCompanionAudioPath(
videoPath: string,
sourceAudioPath: string,
suffix: MacCompanionAudioSuffix,
) {
const separatorIndex = Math.max(videoPath.lastIndexOf("/"), videoPath.lastIndexOf("\\"));
const videoDirectory = separatorIndex >= 0 ? videoPath.slice(0, separatorIndex + 1) : "";
const videoFileName = separatorIndex >= 0 ? videoPath.slice(separatorIndex + 1) : videoPath;
const videoPathWithoutExt = `${videoDirectory}${path.parse(videoFileName).name}`;
const sourceExtension = path.extname(sourceAudioPath).toLowerCase() || ".m4a";
return `${videoPathWithoutExt}.${suffix}${sourceExtension}`;
}
@@ -1,20 +1,13 @@
import type { Span } from "dnd-timeline";
import { Plus } from "@phosphor-icons/react";
import {
forwardRef,
useEffect,
useMemo,
useRef,
useState,
} from "react";
import { useScopedT } from "@/contexts/I18nContext";
import { useShortcuts } from "@/contexts/ShortcutsContext";
import { fromFileUrl } from "../projectPersistence";
import type { Span } from "dnd-timeline";
import { forwardRef, useEffect, useMemo, useRef, useState } from "react";
import type {
SourceAudioTrackMeta,
SourceAudioTrackSettings,
SourceAudioTrackWithPeaks,
} from "@/components/video-editor/audio/audioTypes";
import { useScopedT } from "@/contexts/I18nContext";
import { useShortcuts } from "@/contexts/ShortcutsContext";
import { fromFileUrl } from "../projectPersistence";
import type {
AnnotationRegion,
AudioRegion,
@@ -26,12 +19,16 @@ import type {
ZoomRegion,
} from "../types";
import KeyframeMarkers from "./components/markers/KeyframeMarkers";
import TimelineCanvas from "./components/viewport/TimelineCanvas";
import TimelineWrapper from "./components/wrapper/TimelineWrapper";
import { useTimelineAudioPeaks } from "./hooks/useTimelineAudioPeaks";
import { calculateTimelineScale } from "./core/time";
import { useTimelineAudioPeaks } from "./hooks/useTimelineAudioPeaks";
import { useTimelineEditorRuntime } from "./hooks/useTimelineEditorRuntime";
import { useTimelineRange } from "./hooks/useTimelineRange";
import TimelineCanvas from "./components/viewport/TimelineCanvas";
import {
buildSourceSidecarPathCandidates,
buildTimelineSourceAudioTracks,
} from "./sourceAudioTracks";
export interface TimelineEditorProps {
videoDuration: number;
@@ -77,9 +74,7 @@ export interface TimelineEditorProps {
showSourceAudioTrack?: boolean;
onSourceAudioAvailabilityChange?: (available: boolean) => void;
sourceAudioTrackSettings?: SourceAudioTrackSettings;
getSourceAudioTrackSettingsForClip?: (
clipId: string | null,
) => SourceAudioTrackSettings;
getSourceAudioTrackSettingsForClip?: (clipId: string | null) => SourceAudioTrackSettings;
onSourceAudioTracksMetaChange?: (tracks: SourceAudioTrackMeta) => void;
}
@@ -98,16 +93,6 @@ function extractLocalPathFromMediaServerUrl(input: string | null | undefined): s
}
}
function buildSourceSidecarPath(source: string, suffix: "mic" | "system"): string {
const normalized = source.replace(/\\/g, "/");
const lastSlash = normalized.lastIndexOf("/");
const dir = lastSlash >= 0 ? normalized.slice(0, lastSlash + 1) : "";
const fileName = lastSlash >= 0 ? normalized.slice(lastSlash + 1) : normalized;
const dotIndex = fileName.lastIndexOf(".");
const baseName = dotIndex > 0 ? fileName.slice(0, dotIndex) : fileName;
return `${dir}${baseName}.${suffix}.wav`;
}
export interface TimelineEditorHandle {
addZoom: () => void;
suggestZooms: () => void;
@@ -117,7 +102,6 @@ export interface TimelineEditorHandle {
keyframes: { id: string; time: number }[];
}
const TimelineEditor = forwardRef<TimelineEditorHandle, TimelineEditorProps>(
function TimelineEditor(
{
@@ -209,9 +193,7 @@ const TimelineEditor = forwardRef<TimelineEditorHandle, TimelineEditorProps>(
...(newStart > oldClip.startMs
? [{ startMs: oldClip.startMs, endMs: newStart }]
: []),
...(newEnd < oldClip.endMs
? [{ startMs: newEnd, endMs: oldClip.endMs }]
: []),
...(newEnd < oldClip.endMs ? [{ startMs: newEnd, endMs: oldClip.endMs }] : []),
];
const startDelta = newStart - oldClip.startMs;
@@ -245,9 +227,8 @@ const TimelineEditor = forwardRef<TimelineEditorHandle, TimelineEditorProps>(
return { previewSpans, hiddenZoomIds };
}, [clipRegions, liveSpanPreviewById, zoomRegions]);
const { shortcuts: keyShortcuts, isMac } = useShortcuts();
const { peaks: sourceAudioPeaks, loading: sourceAudioLoading } = useTimelineAudioPeaks(videoPath, {
enableSourceSidecarFallback: true,
});
const { peaks: sourceAudioPeaks, loading: sourceAudioLoading } =
useTimelineAudioPeaks(videoPath);
const localSourcePath = useMemo(() => {
if (!videoPath) return null;
return (
@@ -255,56 +236,67 @@ const TimelineEditor = forwardRef<TimelineEditorHandle, TimelineEditorProps>(
(/^file:\/\//i.test(videoPath) ? fromFileUrl(videoPath) : videoPath)
);
}, [videoPath]);
const micSidecarPath = useMemo(
() => (localSourcePath ? buildSourceSidecarPath(localSourcePath, "mic") : null),
const micSidecarPaths = useMemo(
() => (localSourcePath ? buildSourceSidecarPathCandidates(localSourcePath, "mic") : []),
[localSourcePath],
);
const systemSidecarPath = useMemo(
() => (localSourcePath ? buildSourceSidecarPath(localSourcePath, "system") : null),
const micSidecarFallbackPaths = useMemo(() => micSidecarPaths.slice(1), [micSidecarPaths]);
const systemSidecarPaths = useMemo(
() =>
localSourcePath ? buildSourceSidecarPathCandidates(localSourcePath, "system") : [],
[localSourcePath],
);
const { peaks: micSidecarPeaks, loading: micSidecarLoading } = useTimelineAudioPeaks(micSidecarPath);
const { peaks: systemSidecarPeaks, loading: systemSidecarLoading } = useTimelineAudioPeaks(systemSidecarPath);
const sourceAudioTracks = useMemo<SourceAudioTrackWithPeaks[]>(() => {
if (systemSidecarPeaks || micSidecarPeaks) {
const tracks: SourceAudioTrackWithPeaks[] = [];
if (systemSidecarPeaks)
tracks.push({
id: "system",
label: t("audio.systemLabel", "Source System"),
peaks: systemSidecarPeaks,
});
if (micSidecarPeaks)
tracks.push({
id: "mic",
label: t("audio.micLabel", "Source Mic"),
peaks: micSidecarPeaks,
});
return tracks;
}
return sourceAudioPeaks
? [
{
id: "mixed",
label: t("audio.mixedLabel", "Source"),
peaks: sourceAudioPeaks,
},
]
: [];
}, [micSidecarPeaks, sourceAudioPeaks, systemSidecarPeaks, t]);
const systemSidecarFallbackPaths = useMemo(
() => systemSidecarPaths.slice(1),
[systemSidecarPaths],
);
const { peaks: micSidecarPeaks, loading: micSidecarLoading } = useTimelineAudioPeaks(
micSidecarPaths[0] ?? null,
{ fallbackResources: micSidecarFallbackPaths },
);
const { peaks: systemSidecarPeaks, loading: systemSidecarLoading } = useTimelineAudioPeaks(
systemSidecarPaths[0] ?? null,
{
fallbackResources: systemSidecarFallbackPaths,
},
);
const sourceAudioTracks = useMemo(
() =>
buildTimelineSourceAudioTracks({
sourceAudioPeaks,
micSidecarPeaks,
systemSidecarPeaks,
labels: {
system: t("audio.systemLabel", "Source System"),
mic: t("audio.micLabel", "Source Mic"),
mixed: t("audio.mixedLabel", "Source"),
},
}),
[micSidecarPeaks, sourceAudioPeaks, systemSidecarPeaks, t],
);
const isLoading = useMemo(() => {
// If we are still actively trying to load audio peaks (main or sidecars)
if (videoPath && (sourceAudioLoading || micSidecarLoading || systemSidecarLoading)) return true;
if (videoPath && (sourceAudioLoading || micSidecarLoading || systemSidecarLoading))
return true;
// Robust telemetry loading detection:
// If a source path is set but telemetry hasn't arrived (or failed/retried) for it yet.
if (videoSourcePath && cursorTelemetrySourcePath !== videoSourcePath) return true;
return false;
}, [videoPath, videoSourcePath, cursorTelemetrySourcePath, sourceAudioLoading, micSidecarLoading, systemSidecarLoading]);
}, [
videoPath,
videoSourcePath,
cursorTelemetrySourcePath,
sourceAudioLoading,
micSidecarLoading,
systemSidecarLoading,
]);
useEffect(() => {
onSourceAudioTracksMetaChange?.(sourceAudioTracks.map((t) => ({ id: t.id, label: t.label })));
onSourceAudioTracksMetaChange?.(
sourceAudioTracks.map((t) => ({ id: t.id, label: t.label })),
);
}, [onSourceAudioTracksMetaChange, sourceAudioTracks]);
void sourceAudioTrackSettings;
useEffect(() => {
@@ -1,10 +1,12 @@
import { useEffect, useRef, useState } from "react";
import { resolveMediaResourceUrl } from "@/lib/exporter/localMediaSource";
import { fromFileUrl } from "../../projectPersistence";
import { waveformGenerator } from "../../audio/waveform/WaveformGenerator";
import { fromFileUrl } from "../../projectPersistence";
import { WAVEFORM_DEFAULT_PEAK_COUNT } from "../core/constants";
import type { AudioPeaksData } from "../core/timelineTypes";
const EMPTY_FALLBACK_RESOURCES: string[] = [];
function buildSidecarAudioCandidates(sourcePath: string): string[] {
const normalized = sourcePath.replace(/\\/g, "/");
const lastSlash = normalized.lastIndexOf("/");
@@ -18,6 +20,8 @@ function buildSidecarAudioCandidates(sourcePath: string): string[] {
`${dir}${baseName}.mic.wav`,
`${dir}${baseName}.system.m4a`,
`${dir}${baseName}.mic.m4a`,
`${dir}${baseName}.system.webm`,
`${dir}${baseName}.mic.webm`,
];
}
@@ -37,6 +41,7 @@ function extractLocalPathFromMediaServerUrl(input: string): string | null {
interface TimelineAudioPeaksOptions {
enableSourceSidecarFallback?: boolean;
fallbackResources?: string[];
peakCount?: number;
}
@@ -53,6 +58,7 @@ export function useTimelineAudioPeaks(
const [loading, setLoading] = useState(false);
const sourceRef = useRef(mediaResource);
const enableSourceSidecarFallback = options.enableSourceSidecarFallback ?? false;
const fallbackResources = options.fallbackResources ?? EMPTY_FALLBACK_RESOURCES;
const peakCount = options.peakCount ?? WAVEFORM_DEFAULT_PEAK_COUNT;
useEffect(() => {
@@ -83,25 +89,29 @@ export function useTimelineAudioPeaks(
// fallthrough
}
if (!enableSourceSidecarFallback) {
if (!enableSourceSidecarFallback && fallbackResources.length === 0) {
if (!cancelled && sourceRef.current === mediaResource) {
setLoading(false);
}
return;
}
const localPathFromServer = extractLocalPathFromMediaServerUrl(mediaResource);
const localSourcePath =
localPathFromServer ||
(/^file:\/\//i.test(mediaResource) ? fromFileUrl(mediaResource) : mediaResource);
if (!localSourcePath) {
if (!cancelled && sourceRef.current === mediaResource) {
setLoading(false);
let sourceSidecarCandidates: string[] = [];
if (enableSourceSidecarFallback) {
const localPathFromServer = extractLocalPathFromMediaServerUrl(mediaResource);
const localSourcePath =
localPathFromServer ||
(/^file:\/\//i.test(mediaResource)
? fromFileUrl(mediaResource)
: mediaResource);
if (localSourcePath) {
sourceSidecarCandidates = buildSidecarAudioCandidates(localSourcePath);
}
return;
}
const candidates = buildSidecarAudioCandidates(localSourcePath);
const candidates = Array.from(
new Set([...fallbackResources, ...sourceSidecarCandidates]),
);
for (const candidate of candidates) {
try {
const result = await tryGenerate(candidate);
@@ -125,7 +135,7 @@ export function useTimelineAudioPeaks(
return () => {
cancelled = true;
};
}, [mediaResource, enableSourceSidecarFallback, peakCount]);
}, [mediaResource, enableSourceSidecarFallback, fallbackResources, peakCount]);
return { peaks, loading };
}
@@ -0,0 +1,90 @@
import { describe, expect, it } from "vitest";
import type { AudioPeaksData } from "./core/timelineTypes";
import {
buildSourceSidecarPathCandidates,
buildTimelineSourceAudioTracks,
} from "./sourceAudioTracks";
function peaks(id: number): AudioPeaksData {
return {
durationMs: 1000,
peaks: new Float32Array([id]),
};
}
const labels = {
system: "Source System",
mic: "Source Mic",
mixed: "Source",
};
describe("timeline source audio tracks", () => {
it("builds candidates for Windows and macOS sidecar containers", () => {
expect(buildSourceSidecarPathCandidates("C:\\Recordly\\recording-1.mp4", "mic")).toEqual([
"C:/Recordly/recording-1.mic.wav",
"C:/Recordly/recording-1.mic.m4a",
"C:/Recordly/recording-1.mic.webm",
]);
});
it("keeps embedded system audio controllable when mic is a sidecar", () => {
const source = peaks(1);
const mic = peaks(2);
expect(
buildTimelineSourceAudioTracks({
sourceAudioPeaks: source,
micSidecarPeaks: mic,
systemSidecarPeaks: null,
labels,
}),
).toEqual([
{ id: "system", label: "Source System", peaks: source },
{ id: "mic", label: "Source Mic", peaks: mic },
]);
});
it("does not invent a system track when only the mic sidecar exists", () => {
const mic = peaks(2);
expect(
buildTimelineSourceAudioTracks({
sourceAudioPeaks: null,
micSidecarPeaks: mic,
systemSidecarPeaks: null,
labels,
}),
).toEqual([{ id: "mic", label: "Source Mic", peaks: mic }]);
});
it("uses dedicated sidecars over the embedded track when both source tracks exist", () => {
const source = peaks(1);
const system = peaks(2);
const mic = peaks(3);
expect(
buildTimelineSourceAudioTracks({
sourceAudioPeaks: source,
micSidecarPeaks: mic,
systemSidecarPeaks: system,
labels,
}),
).toEqual([
{ id: "system", label: "Source System", peaks: system },
{ id: "mic", label: "Source Mic", peaks: mic },
]);
});
it("falls back to one mixed source track when no dedicated sidecar exists", () => {
const source = peaks(1);
expect(
buildTimelineSourceAudioTracks({
sourceAudioPeaks: source,
micSidecarPeaks: null,
systemSidecarPeaks: null,
labels,
}),
).toEqual([{ id: "mixed", label: "Source", peaks: source }]);
});
});
@@ -0,0 +1,68 @@
import type { SourceAudioTrackWithPeaks } from "@/components/video-editor/audio/audioTypes";
import type { AudioPeaksData } from "./core/timelineTypes";
const SOURCE_SIDECAR_EXTENSIONS = [".wav", ".m4a", ".webm"] as const;
export function buildSourceSidecarPathCandidates(
source: string,
suffix: "mic" | "system",
): string[] {
const normalized = source.replace(/\\/g, "/");
const lastSlash = normalized.lastIndexOf("/");
const dir = lastSlash >= 0 ? normalized.slice(0, lastSlash + 1) : "";
const fileName = lastSlash >= 0 ? normalized.slice(lastSlash + 1) : normalized;
const dotIndex = fileName.lastIndexOf(".");
const baseName = dotIndex > 0 ? fileName.slice(0, dotIndex) : fileName;
return SOURCE_SIDECAR_EXTENSIONS.map((extension) => `${dir}${baseName}.${suffix}${extension}`);
}
export function buildTimelineSourceAudioTracks({
sourceAudioPeaks,
micSidecarPeaks,
systemSidecarPeaks,
labels,
}: {
sourceAudioPeaks: AudioPeaksData | null;
micSidecarPeaks: AudioPeaksData | null;
systemSidecarPeaks: AudioPeaksData | null;
labels: {
system: string;
mic: string;
mixed: string;
};
}): SourceAudioTrackWithPeaks[] {
if (systemSidecarPeaks || micSidecarPeaks) {
const tracks: SourceAudioTrackWithPeaks[] = [];
if (systemSidecarPeaks) {
tracks.push({
id: "system",
label: labels.system,
peaks: systemSidecarPeaks,
});
} else if (micSidecarPeaks && sourceAudioPeaks) {
tracks.push({
id: "system",
label: labels.system,
peaks: sourceAudioPeaks,
});
}
if (micSidecarPeaks) {
tracks.push({
id: "mic",
label: labels.mic,
peaks: micSidecarPeaks,
});
}
return tracks;
}
return sourceAudioPeaks
? [
{
id: "mixed",
label: labels.mixed,
peaks: sourceAudioPeaks,
},
]
: [];
}
+92 -1
View File
@@ -1,6 +1,6 @@
import { describe, expect, it, vi } from "vitest";
import { AudioProcessor } from "./audioEncoder";
import { AudioProcessor, softLimitOfflineMixPeaksInPlace } from "./audioEncoder";
type OfflineRenderTestHarness = AudioProcessor & {
decodeAudioFromUrl(url: string): Promise<AudioBuffer | null>;
@@ -26,8 +26,32 @@ type OfflineRenderTestHarness = AudioProcessor & {
sourceAudioFallbackStartDelayMsByPath: Record<string, number> | undefined,
muxer: unknown,
): Promise<void>;
renderChunked(
prepared: {
mainBufferEntry: null;
companionEntries: [];
regionEntries: [];
mutedSourceOutputRangesSec: [];
slices: [];
outputDurationMs: number;
numChannels: number;
},
totalOutputSec: number,
onChunk: (
rendered: AudioBuffer,
outputOffsetSec: number,
chunkIndex: number,
) => Promise<void>,
): Promise<void>;
};
function fakeAudioBuffer(channels: Float32Array[]): AudioBuffer {
return {
numberOfChannels: channels.length,
getChannelData: (channel: number) => channels[channel],
} as AudioBuffer;
}
describe("AudioProcessor offline render preparation", () => {
it("keeps embedded source audio separate from external companion sidecars", async () => {
const processor = new AudioProcessor() as unknown as OfflineRenderTestHarness;
@@ -136,4 +160,71 @@ describe("AudioProcessor offline render preparation", () => {
expect(loadAudioFileDemuxer).not.toHaveBeenCalled();
expect(renderAndMuxOfflineAudio).toHaveBeenCalled();
});
it("soft-limits mixed peaks before encoding or WAV conversion", () => {
const samples = new Float32Array([
-1.6,
-0.5,
Number.NEGATIVE_INFINITY,
Number.NaN,
0,
0.5,
0.95,
Number.POSITIVE_INFINITY,
1.6,
]);
const changed = softLimitOfflineMixPeaksInPlace(fakeAudioBuffer([samples]));
expect(changed).toBe(true);
expect(samples[0]).toBeGreaterThanOrEqual(-0.986);
expect(samples[1]).toBe(-0.5);
expect(samples[2]).toBe(0);
expect(samples[3]).toBe(0);
expect(samples[5]).toBe(0.5);
expect(samples[6]).toBeLessThan(0.95);
expect(samples[6]).toBeGreaterThan(0.9);
expect(samples[7]).toBe(0);
expect(samples[8]).toBeLessThanOrEqual(0.986);
});
it("runs the offline mix limiter for every rendered chunk", async () => {
const processor = new AudioProcessor() as unknown as OfflineRenderTestHarness;
const renderedSamples = new Float32Array([1.4]);
const renderedBuffer = fakeAudioBuffer([renderedSamples]);
const originalOfflineAudioContext = globalThis.OfflineAudioContext;
(
globalThis as unknown as { OfflineAudioContext: typeof OfflineAudioContext }
).OfflineAudioContext = class {
constructor() {}
startRendering() {
return Promise.resolve(renderedBuffer);
}
} as unknown as typeof OfflineAudioContext;
try {
let observedPeak = Number.POSITIVE_INFINITY;
await processor.renderChunked(
{
mainBufferEntry: null,
companionEntries: [],
regionEntries: [],
mutedSourceOutputRangesSec: [],
slices: [],
outputDurationMs: 100,
numChannels: 1,
},
0.1,
async (rendered) => {
observedPeak = rendered.getChannelData(0)[0] ?? 0;
},
);
expect(observedPeak).toBeLessThanOrEqual(0.986);
} finally {
(
globalThis as unknown as { OfflineAudioContext: typeof OfflineAudioContext }
).OfflineAudioContext = originalOfflineAudioContext;
}
});
});
+69 -17
View File
@@ -1,20 +1,17 @@
import { WebDemuxer } from "web-demuxer";
import { SOURCE_AUDIO_NORMALIZE_GAIN } from "@/components/video-editor/audio/audioTypes";
import type {
AudioRegion,
ClipRegion,
SpeedRegion,
SourceAudioTrackSettings,
SpeedRegion,
TrimRegion,
} from "@/components/video-editor/types";
import {
buildResolvedAudioPlan,
SourceTrackId,
} from "@/lib/exporter/audioRoutingEngine";
import { buildResolvedAudioPlan, SourceTrackId } from "@/lib/exporter/audioRoutingEngine";
import { estimateCompanionAudioStartDelaySeconds } from "@/lib/mediaTiming";
import { resolveMediaElementSource } from "./localMediaSource";
import type { VideoMuxer } from "./muxer";
import { resolveSourceTrackRoutingPolicy } from "./sourceTrackRoutingPolicy";
import { SOURCE_AUDIO_NORMALIZE_GAIN } from "@/components/video-editor/audio/audioTypes";
const AUDIO_BITRATE = 128_000;
const DECODE_BACKPRESSURE_LIMIT = 20;
@@ -24,6 +21,44 @@ const MP4_AUDIO_CODEC = "mp4a.40.2";
const OFFLINE_AUDIO_SAMPLE_RATE = 48_000;
const OFFLINE_ENCODE_CHUNK_FRAMES = 1024;
const OFFLINE_CHUNK_DURATION_SEC = 30;
const OFFLINE_MIX_SOFT_LIMITER_THRESHOLD = 0.9;
const OFFLINE_MIX_SOFT_LIMITER_CEILING = 0.985;
function softLimitSample(sample: number): number {
const magnitude = Math.abs(sample);
if (magnitude <= OFFLINE_MIX_SOFT_LIMITER_THRESHOLD) {
return sample;
}
const sign = sample < 0 ? -1 : 1;
const kneeRange = 1 - OFFLINE_MIX_SOFT_LIMITER_THRESHOLD;
const limitedMagnitude =
OFFLINE_MIX_SOFT_LIMITER_THRESHOLD +
kneeRange * Math.tanh((magnitude - OFFLINE_MIX_SOFT_LIMITER_THRESHOLD) / kneeRange);
return sign * Math.min(OFFLINE_MIX_SOFT_LIMITER_CEILING, limitedMagnitude);
}
export function softLimitOfflineMixPeaksInPlace(buffer: AudioBuffer): boolean {
let changed = false;
for (let channel = 0; channel < buffer.numberOfChannels; channel += 1) {
const data = buffer.getChannelData(channel);
for (let index = 0; index < data.length; index += 1) {
const sample = data[index];
if (!Number.isFinite(sample)) {
data[index] = 0;
changed = true;
continue;
}
const limited = softLimitSample(sample);
if (limited !== sample) {
data[index] = limited;
changed = true;
}
}
}
return changed;
}
function resolveSourceTrackGain(
sourceAudioTrackSettings: SourceAudioTrackSettings | undefined,
@@ -698,8 +733,11 @@ export class AudioProcessor {
if (this.cancelled) throw new Error("Export cancelled");
// Decode companion / sidecar audio files
const companionEntries: Array<{ buffer: AudioBuffer; startDelaySec: number; gain: number }> =
[];
const companionEntries: Array<{
buffer: AudioBuffer;
startDelaySec: number;
gain: number;
}> = [];
const refDuration =
mainBuffer?.duration ??
(resolvedPlan.playbackPaths.length > 0 ? await this.getMediaDurationSec(videoUrl) : 0);
@@ -947,6 +985,7 @@ export class AudioProcessor {
const rendered = await offlineCtx.startRendering();
if (this.cancelled) break;
softLimitOfflineMixPeaksInPlace(rendered);
await onChunk(rendered, outputOffsetSec, i);
@@ -1459,7 +1498,9 @@ export class AudioProcessor {
{
startSec: localOutputStartSec + chunkOutputStartSec,
endSec:
localOutputStartSec + chunkOutputStartSec + effectiveSourceDurationSec / slice.speed,
localOutputStartSec +
chunkOutputStartSec +
effectiveSourceDurationSec / slice.speed,
},
];
for (const mutedRange of mutedOutputRangesSec) {
@@ -1491,7 +1532,8 @@ export class AudioProcessor {
const sourceOffsetSec =
effectiveBufferStartSec +
(audibleRange.startSec - (localOutputStartSec + chunkOutputStartSec)) * slice.speed;
(audibleRange.startSec - (localOutputStartSec + chunkOutputStartSec)) *
slice.speed;
const localStartSec = audibleRange.startSec - chunkOutputStartSec;
const sourceDurationSec = audibleDurationSec * slice.speed;
@@ -1543,7 +1585,9 @@ export class AudioProcessor {
if (copyLength > 0) {
for (let c = 0; c < channels; c++) {
outBuffer.copyToChannel(
originalBuffer.getChannelData(c).subarray(startSample, startSample + copyLength),
originalBuffer
.getChannelData(c)
.subarray(startSample, startSample + copyLength),
c,
);
}
@@ -1559,7 +1603,7 @@ export class AudioProcessor {
const workStartIn = Math.max(0, startSample - paddingInSamples);
const workEndIn = Math.min(originalBuffer.length, endSample + paddingInSamples);
const actualPaddingInStart = startSample - workStartIn;
// We expect the output offset for the requested start to be roughly:
const actualPaddingOutStart = Math.floor(actualPaddingInStart / speed);
@@ -1572,8 +1616,12 @@ export class AudioProcessor {
const workOutSamples = Math.floor((workEndIn - workStartIn) / speed) + windowSize * 2;
const workOutBuffer = ctx.createBuffer(channels, workOutSamples, sampleRate);
const inDataByChannel = Array.from({ length: channels }, (_, c) => originalBuffer.getChannelData(c));
const workOutDataByChannel = Array.from({ length: channels }, (_, c) => workOutBuffer.getChannelData(c));
const inDataByChannel = Array.from({ length: channels }, (_, c) =>
originalBuffer.getChannelData(c),
);
const workOutDataByChannel = Array.from({ length: channels }, (_, c) =>
workOutBuffer.getChannelData(c),
);
const window = new Float32Array(windowSize);
for (let i = 0; i < windowSize; i++) {
@@ -1587,7 +1635,8 @@ export class AudioProcessor {
for (let i = 0; i < windowSize; i++) {
if (inOffset + i < workEndIn && outOffset + i < workOutSamples) {
for (let c = 0; c < channels; c++) {
workOutDataByChannel[c][outOffset + i] += inDataByChannel[c][inOffset + i] * window[i];
workOutDataByChannel[c][outOffset + i] +=
inDataByChannel[c][inOffset + i] * window[i];
}
}
}
@@ -1609,7 +1658,9 @@ export class AudioProcessor {
for (let i = 0; i < hopOut; i += 4) {
if (outOffset + i < workOutSamples && testOffset + i < workEndIn) {
for (let c = 0; c < channels; c++) {
corr += workOutDataByChannel[c][outOffset + i] * inDataByChannel[c][testOffset + i];
corr +=
workOutDataByChannel[c][outOffset + i] *
inDataByChannel[c][testOffset + i];
}
}
}
@@ -1624,7 +1675,8 @@ export class AudioProcessor {
for (let i = 0; i < windowSize; i++) {
if (bestOffset + i < workEndIn && outOffset + i < workOutSamples) {
for (let c = 0; c < channels; c++) {
workOutDataByChannel[c][outOffset + i] += inDataByChannel[c][bestOffset + i] * window[i];
workOutDataByChannel[c][outOffset + i] +=
inDataByChannel[c][bestOffset + i] * window[i];
}
}
}