mirror of
https://github.com/webadderallorg/Recordly.git
synced 2026-09-24 14:55:37 +00:00
fix: preserve mac audio sidecar tracks
This commit is contained in:
@@ -30,9 +30,9 @@ import {
|
||||
validateRecordedVideo,
|
||||
} from "./diagnostics";
|
||||
import { emitRecordingInterrupted } from "./events";
|
||||
import { getFinalMacCompanionAudioPath } from "./macCompanionAudio";
|
||||
import { pruneAutoRecordings } from "./prune";
|
||||
|
||||
|
||||
export function waitForNativeCaptureStart(process: ChildProcessWithoutNullStreams) {
|
||||
return new Promise<void>((resolve, reject) => {
|
||||
const timer = setTimeout(() => {
|
||||
@@ -119,13 +119,11 @@ export async function muxNativeMacRecordingWithAudio(
|
||||
microphonePath?: string | null,
|
||||
) {
|
||||
console.log("[mac-mux] Optimization active: keeping tracks separate.");
|
||||
|
||||
const videoPathWithoutExt = videoPath.replace(/\.[^.]+$/u, "");
|
||||
|
||||
// Optimization: instead of heavy FFmpeg muxing, we ensure audio sidecars
|
||||
// are available alongside the video for the editor.
|
||||
if (systemAudioPath) {
|
||||
const finalSystemPath = `${videoPathWithoutExt}.system.wav`;
|
||||
const finalSystemPath = getFinalMacCompanionAudioPath(videoPath, systemAudioPath, "system");
|
||||
try {
|
||||
const stat = await fs.stat(systemAudioPath);
|
||||
if (stat.size > 0 && systemAudioPath !== finalSystemPath) {
|
||||
@@ -137,7 +135,7 @@ export async function muxNativeMacRecordingWithAudio(
|
||||
}
|
||||
|
||||
if (microphonePath) {
|
||||
const finalMicPath = `${videoPathWithoutExt}.mic.wav`;
|
||||
const finalMicPath = getFinalMacCompanionAudioPath(videoPath, microphonePath, "mic");
|
||||
try {
|
||||
const stat = await fs.stat(microphonePath);
|
||||
if (stat.size > 0 && microphonePath !== finalMicPath) {
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { getFinalMacCompanionAudioPath } from "./macCompanionAudio";
|
||||
|
||||
describe("mac companion audio paths", () => {
|
||||
it("preserves the helper's AAC container extension", () => {
|
||||
expect(
|
||||
getFinalMacCompanionAudioPath(
|
||||
"/Users/egg/Recordly/recording-1.mp4",
|
||||
"/Users/egg/Recordly/recording-1.mic.m4a",
|
||||
"mic",
|
||||
),
|
||||
).toBe("/Users/egg/Recordly/recording-1.mic.m4a");
|
||||
});
|
||||
|
||||
it("preserves legacy sidecar extensions instead of renaming bytes", () => {
|
||||
expect(
|
||||
getFinalMacCompanionAudioPath(
|
||||
"/Users/egg/Recordly/recording-1.mp4",
|
||||
"/tmp/recordly-native.system.webm",
|
||||
"system",
|
||||
),
|
||||
).toBe("/Users/egg/Recordly/recording-1.system.webm");
|
||||
});
|
||||
|
||||
it("keeps dotted directories when the video path has no extension", () => {
|
||||
expect(
|
||||
getFinalMacCompanionAudioPath(
|
||||
"/Users/egg/Recordly.videos/recording-1",
|
||||
"/tmp/recordly-native.mic.m4a",
|
||||
"mic",
|
||||
),
|
||||
).toBe("/Users/egg/Recordly.videos/recording-1.mic.m4a");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,16 @@
|
||||
import path from "node:path";
|
||||
|
||||
export type MacCompanionAudioSuffix = "system" | "mic";
|
||||
|
||||
export function getFinalMacCompanionAudioPath(
|
||||
videoPath: string,
|
||||
sourceAudioPath: string,
|
||||
suffix: MacCompanionAudioSuffix,
|
||||
) {
|
||||
const separatorIndex = Math.max(videoPath.lastIndexOf("/"), videoPath.lastIndexOf("\\"));
|
||||
const videoDirectory = separatorIndex >= 0 ? videoPath.slice(0, separatorIndex + 1) : "";
|
||||
const videoFileName = separatorIndex >= 0 ? videoPath.slice(separatorIndex + 1) : videoPath;
|
||||
const videoPathWithoutExt = `${videoDirectory}${path.parse(videoFileName).name}`;
|
||||
const sourceExtension = path.extname(sourceAudioPath).toLowerCase() || ".m4a";
|
||||
return `${videoPathWithoutExt}.${suffix}${sourceExtension}`;
|
||||
}
|
||||
@@ -1,20 +1,13 @@
|
||||
import type { Span } from "dnd-timeline";
|
||||
import { Plus } from "@phosphor-icons/react";
|
||||
import {
|
||||
forwardRef,
|
||||
useEffect,
|
||||
useMemo,
|
||||
useRef,
|
||||
useState,
|
||||
} from "react";
|
||||
import { useScopedT } from "@/contexts/I18nContext";
|
||||
import { useShortcuts } from "@/contexts/ShortcutsContext";
|
||||
import { fromFileUrl } from "../projectPersistence";
|
||||
import type { Span } from "dnd-timeline";
|
||||
import { forwardRef, useEffect, useMemo, useRef, useState } from "react";
|
||||
import type {
|
||||
SourceAudioTrackMeta,
|
||||
SourceAudioTrackSettings,
|
||||
SourceAudioTrackWithPeaks,
|
||||
} from "@/components/video-editor/audio/audioTypes";
|
||||
import { useScopedT } from "@/contexts/I18nContext";
|
||||
import { useShortcuts } from "@/contexts/ShortcutsContext";
|
||||
import { fromFileUrl } from "../projectPersistence";
|
||||
import type {
|
||||
AnnotationRegion,
|
||||
AudioRegion,
|
||||
@@ -26,12 +19,16 @@ import type {
|
||||
ZoomRegion,
|
||||
} from "../types";
|
||||
import KeyframeMarkers from "./components/markers/KeyframeMarkers";
|
||||
import TimelineCanvas from "./components/viewport/TimelineCanvas";
|
||||
import TimelineWrapper from "./components/wrapper/TimelineWrapper";
|
||||
import { useTimelineAudioPeaks } from "./hooks/useTimelineAudioPeaks";
|
||||
import { calculateTimelineScale } from "./core/time";
|
||||
import { useTimelineAudioPeaks } from "./hooks/useTimelineAudioPeaks";
|
||||
import { useTimelineEditorRuntime } from "./hooks/useTimelineEditorRuntime";
|
||||
import { useTimelineRange } from "./hooks/useTimelineRange";
|
||||
import TimelineCanvas from "./components/viewport/TimelineCanvas";
|
||||
import {
|
||||
buildSourceSidecarPathCandidates,
|
||||
buildTimelineSourceAudioTracks,
|
||||
} from "./sourceAudioTracks";
|
||||
|
||||
export interface TimelineEditorProps {
|
||||
videoDuration: number;
|
||||
@@ -77,9 +74,7 @@ export interface TimelineEditorProps {
|
||||
showSourceAudioTrack?: boolean;
|
||||
onSourceAudioAvailabilityChange?: (available: boolean) => void;
|
||||
sourceAudioTrackSettings?: SourceAudioTrackSettings;
|
||||
getSourceAudioTrackSettingsForClip?: (
|
||||
clipId: string | null,
|
||||
) => SourceAudioTrackSettings;
|
||||
getSourceAudioTrackSettingsForClip?: (clipId: string | null) => SourceAudioTrackSettings;
|
||||
onSourceAudioTracksMetaChange?: (tracks: SourceAudioTrackMeta) => void;
|
||||
}
|
||||
|
||||
@@ -98,16 +93,6 @@ function extractLocalPathFromMediaServerUrl(input: string | null | undefined): s
|
||||
}
|
||||
}
|
||||
|
||||
function buildSourceSidecarPath(source: string, suffix: "mic" | "system"): string {
|
||||
const normalized = source.replace(/\\/g, "/");
|
||||
const lastSlash = normalized.lastIndexOf("/");
|
||||
const dir = lastSlash >= 0 ? normalized.slice(0, lastSlash + 1) : "";
|
||||
const fileName = lastSlash >= 0 ? normalized.slice(lastSlash + 1) : normalized;
|
||||
const dotIndex = fileName.lastIndexOf(".");
|
||||
const baseName = dotIndex > 0 ? fileName.slice(0, dotIndex) : fileName;
|
||||
return `${dir}${baseName}.${suffix}.wav`;
|
||||
}
|
||||
|
||||
export interface TimelineEditorHandle {
|
||||
addZoom: () => void;
|
||||
suggestZooms: () => void;
|
||||
@@ -117,7 +102,6 @@ export interface TimelineEditorHandle {
|
||||
keyframes: { id: string; time: number }[];
|
||||
}
|
||||
|
||||
|
||||
const TimelineEditor = forwardRef<TimelineEditorHandle, TimelineEditorProps>(
|
||||
function TimelineEditor(
|
||||
{
|
||||
@@ -209,9 +193,7 @@ const TimelineEditor = forwardRef<TimelineEditorHandle, TimelineEditorProps>(
|
||||
...(newStart > oldClip.startMs
|
||||
? [{ startMs: oldClip.startMs, endMs: newStart }]
|
||||
: []),
|
||||
...(newEnd < oldClip.endMs
|
||||
? [{ startMs: newEnd, endMs: oldClip.endMs }]
|
||||
: []),
|
||||
...(newEnd < oldClip.endMs ? [{ startMs: newEnd, endMs: oldClip.endMs }] : []),
|
||||
];
|
||||
|
||||
const startDelta = newStart - oldClip.startMs;
|
||||
@@ -245,9 +227,8 @@ const TimelineEditor = forwardRef<TimelineEditorHandle, TimelineEditorProps>(
|
||||
return { previewSpans, hiddenZoomIds };
|
||||
}, [clipRegions, liveSpanPreviewById, zoomRegions]);
|
||||
const { shortcuts: keyShortcuts, isMac } = useShortcuts();
|
||||
const { peaks: sourceAudioPeaks, loading: sourceAudioLoading } = useTimelineAudioPeaks(videoPath, {
|
||||
enableSourceSidecarFallback: true,
|
||||
});
|
||||
const { peaks: sourceAudioPeaks, loading: sourceAudioLoading } =
|
||||
useTimelineAudioPeaks(videoPath);
|
||||
const localSourcePath = useMemo(() => {
|
||||
if (!videoPath) return null;
|
||||
return (
|
||||
@@ -255,56 +236,67 @@ const TimelineEditor = forwardRef<TimelineEditorHandle, TimelineEditorProps>(
|
||||
(/^file:\/\//i.test(videoPath) ? fromFileUrl(videoPath) : videoPath)
|
||||
);
|
||||
}, [videoPath]);
|
||||
const micSidecarPath = useMemo(
|
||||
() => (localSourcePath ? buildSourceSidecarPath(localSourcePath, "mic") : null),
|
||||
const micSidecarPaths = useMemo(
|
||||
() => (localSourcePath ? buildSourceSidecarPathCandidates(localSourcePath, "mic") : []),
|
||||
[localSourcePath],
|
||||
);
|
||||
const systemSidecarPath = useMemo(
|
||||
() => (localSourcePath ? buildSourceSidecarPath(localSourcePath, "system") : null),
|
||||
const micSidecarFallbackPaths = useMemo(() => micSidecarPaths.slice(1), [micSidecarPaths]);
|
||||
const systemSidecarPaths = useMemo(
|
||||
() =>
|
||||
localSourcePath ? buildSourceSidecarPathCandidates(localSourcePath, "system") : [],
|
||||
[localSourcePath],
|
||||
);
|
||||
const { peaks: micSidecarPeaks, loading: micSidecarLoading } = useTimelineAudioPeaks(micSidecarPath);
|
||||
const { peaks: systemSidecarPeaks, loading: systemSidecarLoading } = useTimelineAudioPeaks(systemSidecarPath);
|
||||
const sourceAudioTracks = useMemo<SourceAudioTrackWithPeaks[]>(() => {
|
||||
if (systemSidecarPeaks || micSidecarPeaks) {
|
||||
const tracks: SourceAudioTrackWithPeaks[] = [];
|
||||
if (systemSidecarPeaks)
|
||||
tracks.push({
|
||||
id: "system",
|
||||
label: t("audio.systemLabel", "Source System"),
|
||||
peaks: systemSidecarPeaks,
|
||||
});
|
||||
if (micSidecarPeaks)
|
||||
tracks.push({
|
||||
id: "mic",
|
||||
label: t("audio.micLabel", "Source Mic"),
|
||||
peaks: micSidecarPeaks,
|
||||
});
|
||||
return tracks;
|
||||
}
|
||||
return sourceAudioPeaks
|
||||
? [
|
||||
{
|
||||
id: "mixed",
|
||||
label: t("audio.mixedLabel", "Source"),
|
||||
peaks: sourceAudioPeaks,
|
||||
},
|
||||
]
|
||||
: [];
|
||||
}, [micSidecarPeaks, sourceAudioPeaks, systemSidecarPeaks, t]);
|
||||
const systemSidecarFallbackPaths = useMemo(
|
||||
() => systemSidecarPaths.slice(1),
|
||||
[systemSidecarPaths],
|
||||
);
|
||||
const { peaks: micSidecarPeaks, loading: micSidecarLoading } = useTimelineAudioPeaks(
|
||||
micSidecarPaths[0] ?? null,
|
||||
{ fallbackResources: micSidecarFallbackPaths },
|
||||
);
|
||||
const { peaks: systemSidecarPeaks, loading: systemSidecarLoading } = useTimelineAudioPeaks(
|
||||
systemSidecarPaths[0] ?? null,
|
||||
{
|
||||
fallbackResources: systemSidecarFallbackPaths,
|
||||
},
|
||||
);
|
||||
const sourceAudioTracks = useMemo(
|
||||
() =>
|
||||
buildTimelineSourceAudioTracks({
|
||||
sourceAudioPeaks,
|
||||
micSidecarPeaks,
|
||||
systemSidecarPeaks,
|
||||
labels: {
|
||||
system: t("audio.systemLabel", "Source System"),
|
||||
mic: t("audio.micLabel", "Source Mic"),
|
||||
mixed: t("audio.mixedLabel", "Source"),
|
||||
},
|
||||
}),
|
||||
[micSidecarPeaks, sourceAudioPeaks, systemSidecarPeaks, t],
|
||||
);
|
||||
|
||||
const isLoading = useMemo(() => {
|
||||
// If we are still actively trying to load audio peaks (main or sidecars)
|
||||
if (videoPath && (sourceAudioLoading || micSidecarLoading || systemSidecarLoading)) return true;
|
||||
if (videoPath && (sourceAudioLoading || micSidecarLoading || systemSidecarLoading))
|
||||
return true;
|
||||
|
||||
// Robust telemetry loading detection:
|
||||
// If a source path is set but telemetry hasn't arrived (or failed/retried) for it yet.
|
||||
if (videoSourcePath && cursorTelemetrySourcePath !== videoSourcePath) return true;
|
||||
|
||||
return false;
|
||||
}, [videoPath, videoSourcePath, cursorTelemetrySourcePath, sourceAudioLoading, micSidecarLoading, systemSidecarLoading]);
|
||||
}, [
|
||||
videoPath,
|
||||
videoSourcePath,
|
||||
cursorTelemetrySourcePath,
|
||||
sourceAudioLoading,
|
||||
micSidecarLoading,
|
||||
systemSidecarLoading,
|
||||
]);
|
||||
useEffect(() => {
|
||||
onSourceAudioTracksMetaChange?.(sourceAudioTracks.map((t) => ({ id: t.id, label: t.label })));
|
||||
onSourceAudioTracksMetaChange?.(
|
||||
sourceAudioTracks.map((t) => ({ id: t.id, label: t.label })),
|
||||
);
|
||||
}, [onSourceAudioTracksMetaChange, sourceAudioTracks]);
|
||||
void sourceAudioTrackSettings;
|
||||
useEffect(() => {
|
||||
|
||||
@@ -1,10 +1,12 @@
|
||||
import { useEffect, useRef, useState } from "react";
|
||||
import { resolveMediaResourceUrl } from "@/lib/exporter/localMediaSource";
|
||||
import { fromFileUrl } from "../../projectPersistence";
|
||||
import { waveformGenerator } from "../../audio/waveform/WaveformGenerator";
|
||||
import { fromFileUrl } from "../../projectPersistence";
|
||||
import { WAVEFORM_DEFAULT_PEAK_COUNT } from "../core/constants";
|
||||
import type { AudioPeaksData } from "../core/timelineTypes";
|
||||
|
||||
const EMPTY_FALLBACK_RESOURCES: string[] = [];
|
||||
|
||||
function buildSidecarAudioCandidates(sourcePath: string): string[] {
|
||||
const normalized = sourcePath.replace(/\\/g, "/");
|
||||
const lastSlash = normalized.lastIndexOf("/");
|
||||
@@ -18,6 +20,8 @@ function buildSidecarAudioCandidates(sourcePath: string): string[] {
|
||||
`${dir}${baseName}.mic.wav`,
|
||||
`${dir}${baseName}.system.m4a`,
|
||||
`${dir}${baseName}.mic.m4a`,
|
||||
`${dir}${baseName}.system.webm`,
|
||||
`${dir}${baseName}.mic.webm`,
|
||||
];
|
||||
}
|
||||
|
||||
@@ -37,6 +41,7 @@ function extractLocalPathFromMediaServerUrl(input: string): string | null {
|
||||
|
||||
interface TimelineAudioPeaksOptions {
|
||||
enableSourceSidecarFallback?: boolean;
|
||||
fallbackResources?: string[];
|
||||
peakCount?: number;
|
||||
}
|
||||
|
||||
@@ -53,6 +58,7 @@ export function useTimelineAudioPeaks(
|
||||
const [loading, setLoading] = useState(false);
|
||||
const sourceRef = useRef(mediaResource);
|
||||
const enableSourceSidecarFallback = options.enableSourceSidecarFallback ?? false;
|
||||
const fallbackResources = options.fallbackResources ?? EMPTY_FALLBACK_RESOURCES;
|
||||
const peakCount = options.peakCount ?? WAVEFORM_DEFAULT_PEAK_COUNT;
|
||||
|
||||
useEffect(() => {
|
||||
@@ -83,25 +89,29 @@ export function useTimelineAudioPeaks(
|
||||
// fallthrough
|
||||
}
|
||||
|
||||
if (!enableSourceSidecarFallback) {
|
||||
if (!enableSourceSidecarFallback && fallbackResources.length === 0) {
|
||||
if (!cancelled && sourceRef.current === mediaResource) {
|
||||
setLoading(false);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
const localPathFromServer = extractLocalPathFromMediaServerUrl(mediaResource);
|
||||
const localSourcePath =
|
||||
localPathFromServer ||
|
||||
(/^file:\/\//i.test(mediaResource) ? fromFileUrl(mediaResource) : mediaResource);
|
||||
if (!localSourcePath) {
|
||||
if (!cancelled && sourceRef.current === mediaResource) {
|
||||
setLoading(false);
|
||||
let sourceSidecarCandidates: string[] = [];
|
||||
if (enableSourceSidecarFallback) {
|
||||
const localPathFromServer = extractLocalPathFromMediaServerUrl(mediaResource);
|
||||
const localSourcePath =
|
||||
localPathFromServer ||
|
||||
(/^file:\/\//i.test(mediaResource)
|
||||
? fromFileUrl(mediaResource)
|
||||
: mediaResource);
|
||||
if (localSourcePath) {
|
||||
sourceSidecarCandidates = buildSidecarAudioCandidates(localSourcePath);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
const candidates = buildSidecarAudioCandidates(localSourcePath);
|
||||
const candidates = Array.from(
|
||||
new Set([...fallbackResources, ...sourceSidecarCandidates]),
|
||||
);
|
||||
for (const candidate of candidates) {
|
||||
try {
|
||||
const result = await tryGenerate(candidate);
|
||||
@@ -125,7 +135,7 @@ export function useTimelineAudioPeaks(
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
}, [mediaResource, enableSourceSidecarFallback, peakCount]);
|
||||
}, [mediaResource, enableSourceSidecarFallback, fallbackResources, peakCount]);
|
||||
|
||||
return { peaks, loading };
|
||||
}
|
||||
|
||||
@@ -0,0 +1,90 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import type { AudioPeaksData } from "./core/timelineTypes";
|
||||
import {
|
||||
buildSourceSidecarPathCandidates,
|
||||
buildTimelineSourceAudioTracks,
|
||||
} from "./sourceAudioTracks";
|
||||
|
||||
function peaks(id: number): AudioPeaksData {
|
||||
return {
|
||||
durationMs: 1000,
|
||||
peaks: new Float32Array([id]),
|
||||
};
|
||||
}
|
||||
|
||||
const labels = {
|
||||
system: "Source System",
|
||||
mic: "Source Mic",
|
||||
mixed: "Source",
|
||||
};
|
||||
|
||||
describe("timeline source audio tracks", () => {
|
||||
it("builds candidates for Windows and macOS sidecar containers", () => {
|
||||
expect(buildSourceSidecarPathCandidates("C:\\Recordly\\recording-1.mp4", "mic")).toEqual([
|
||||
"C:/Recordly/recording-1.mic.wav",
|
||||
"C:/Recordly/recording-1.mic.m4a",
|
||||
"C:/Recordly/recording-1.mic.webm",
|
||||
]);
|
||||
});
|
||||
|
||||
it("keeps embedded system audio controllable when mic is a sidecar", () => {
|
||||
const source = peaks(1);
|
||||
const mic = peaks(2);
|
||||
|
||||
expect(
|
||||
buildTimelineSourceAudioTracks({
|
||||
sourceAudioPeaks: source,
|
||||
micSidecarPeaks: mic,
|
||||
systemSidecarPeaks: null,
|
||||
labels,
|
||||
}),
|
||||
).toEqual([
|
||||
{ id: "system", label: "Source System", peaks: source },
|
||||
{ id: "mic", label: "Source Mic", peaks: mic },
|
||||
]);
|
||||
});
|
||||
|
||||
it("does not invent a system track when only the mic sidecar exists", () => {
|
||||
const mic = peaks(2);
|
||||
|
||||
expect(
|
||||
buildTimelineSourceAudioTracks({
|
||||
sourceAudioPeaks: null,
|
||||
micSidecarPeaks: mic,
|
||||
systemSidecarPeaks: null,
|
||||
labels,
|
||||
}),
|
||||
).toEqual([{ id: "mic", label: "Source Mic", peaks: mic }]);
|
||||
});
|
||||
|
||||
it("uses dedicated sidecars over the embedded track when both source tracks exist", () => {
|
||||
const source = peaks(1);
|
||||
const system = peaks(2);
|
||||
const mic = peaks(3);
|
||||
|
||||
expect(
|
||||
buildTimelineSourceAudioTracks({
|
||||
sourceAudioPeaks: source,
|
||||
micSidecarPeaks: mic,
|
||||
systemSidecarPeaks: system,
|
||||
labels,
|
||||
}),
|
||||
).toEqual([
|
||||
{ id: "system", label: "Source System", peaks: system },
|
||||
{ id: "mic", label: "Source Mic", peaks: mic },
|
||||
]);
|
||||
});
|
||||
|
||||
it("falls back to one mixed source track when no dedicated sidecar exists", () => {
|
||||
const source = peaks(1);
|
||||
|
||||
expect(
|
||||
buildTimelineSourceAudioTracks({
|
||||
sourceAudioPeaks: source,
|
||||
micSidecarPeaks: null,
|
||||
systemSidecarPeaks: null,
|
||||
labels,
|
||||
}),
|
||||
).toEqual([{ id: "mixed", label: "Source", peaks: source }]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,68 @@
|
||||
import type { SourceAudioTrackWithPeaks } from "@/components/video-editor/audio/audioTypes";
|
||||
import type { AudioPeaksData } from "./core/timelineTypes";
|
||||
|
||||
const SOURCE_SIDECAR_EXTENSIONS = [".wav", ".m4a", ".webm"] as const;
|
||||
|
||||
export function buildSourceSidecarPathCandidates(
|
||||
source: string,
|
||||
suffix: "mic" | "system",
|
||||
): string[] {
|
||||
const normalized = source.replace(/\\/g, "/");
|
||||
const lastSlash = normalized.lastIndexOf("/");
|
||||
const dir = lastSlash >= 0 ? normalized.slice(0, lastSlash + 1) : "";
|
||||
const fileName = lastSlash >= 0 ? normalized.slice(lastSlash + 1) : normalized;
|
||||
const dotIndex = fileName.lastIndexOf(".");
|
||||
const baseName = dotIndex > 0 ? fileName.slice(0, dotIndex) : fileName;
|
||||
return SOURCE_SIDECAR_EXTENSIONS.map((extension) => `${dir}${baseName}.${suffix}${extension}`);
|
||||
}
|
||||
|
||||
export function buildTimelineSourceAudioTracks({
|
||||
sourceAudioPeaks,
|
||||
micSidecarPeaks,
|
||||
systemSidecarPeaks,
|
||||
labels,
|
||||
}: {
|
||||
sourceAudioPeaks: AudioPeaksData | null;
|
||||
micSidecarPeaks: AudioPeaksData | null;
|
||||
systemSidecarPeaks: AudioPeaksData | null;
|
||||
labels: {
|
||||
system: string;
|
||||
mic: string;
|
||||
mixed: string;
|
||||
};
|
||||
}): SourceAudioTrackWithPeaks[] {
|
||||
if (systemSidecarPeaks || micSidecarPeaks) {
|
||||
const tracks: SourceAudioTrackWithPeaks[] = [];
|
||||
if (systemSidecarPeaks) {
|
||||
tracks.push({
|
||||
id: "system",
|
||||
label: labels.system,
|
||||
peaks: systemSidecarPeaks,
|
||||
});
|
||||
} else if (micSidecarPeaks && sourceAudioPeaks) {
|
||||
tracks.push({
|
||||
id: "system",
|
||||
label: labels.system,
|
||||
peaks: sourceAudioPeaks,
|
||||
});
|
||||
}
|
||||
if (micSidecarPeaks) {
|
||||
tracks.push({
|
||||
id: "mic",
|
||||
label: labels.mic,
|
||||
peaks: micSidecarPeaks,
|
||||
});
|
||||
}
|
||||
return tracks;
|
||||
}
|
||||
|
||||
return sourceAudioPeaks
|
||||
? [
|
||||
{
|
||||
id: "mixed",
|
||||
label: labels.mixed,
|
||||
peaks: sourceAudioPeaks,
|
||||
},
|
||||
]
|
||||
: [];
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
|
||||
import { AudioProcessor } from "./audioEncoder";
|
||||
import { AudioProcessor, softLimitOfflineMixPeaksInPlace } from "./audioEncoder";
|
||||
|
||||
type OfflineRenderTestHarness = AudioProcessor & {
|
||||
decodeAudioFromUrl(url: string): Promise<AudioBuffer | null>;
|
||||
@@ -26,8 +26,32 @@ type OfflineRenderTestHarness = AudioProcessor & {
|
||||
sourceAudioFallbackStartDelayMsByPath: Record<string, number> | undefined,
|
||||
muxer: unknown,
|
||||
): Promise<void>;
|
||||
renderChunked(
|
||||
prepared: {
|
||||
mainBufferEntry: null;
|
||||
companionEntries: [];
|
||||
regionEntries: [];
|
||||
mutedSourceOutputRangesSec: [];
|
||||
slices: [];
|
||||
outputDurationMs: number;
|
||||
numChannels: number;
|
||||
},
|
||||
totalOutputSec: number,
|
||||
onChunk: (
|
||||
rendered: AudioBuffer,
|
||||
outputOffsetSec: number,
|
||||
chunkIndex: number,
|
||||
) => Promise<void>,
|
||||
): Promise<void>;
|
||||
};
|
||||
|
||||
function fakeAudioBuffer(channels: Float32Array[]): AudioBuffer {
|
||||
return {
|
||||
numberOfChannels: channels.length,
|
||||
getChannelData: (channel: number) => channels[channel],
|
||||
} as AudioBuffer;
|
||||
}
|
||||
|
||||
describe("AudioProcessor offline render preparation", () => {
|
||||
it("keeps embedded source audio separate from external companion sidecars", async () => {
|
||||
const processor = new AudioProcessor() as unknown as OfflineRenderTestHarness;
|
||||
@@ -136,4 +160,71 @@ describe("AudioProcessor offline render preparation", () => {
|
||||
expect(loadAudioFileDemuxer).not.toHaveBeenCalled();
|
||||
expect(renderAndMuxOfflineAudio).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("soft-limits mixed peaks before encoding or WAV conversion", () => {
|
||||
const samples = new Float32Array([
|
||||
-1.6,
|
||||
-0.5,
|
||||
Number.NEGATIVE_INFINITY,
|
||||
Number.NaN,
|
||||
0,
|
||||
0.5,
|
||||
0.95,
|
||||
Number.POSITIVE_INFINITY,
|
||||
1.6,
|
||||
]);
|
||||
const changed = softLimitOfflineMixPeaksInPlace(fakeAudioBuffer([samples]));
|
||||
|
||||
expect(changed).toBe(true);
|
||||
expect(samples[0]).toBeGreaterThanOrEqual(-0.986);
|
||||
expect(samples[1]).toBe(-0.5);
|
||||
expect(samples[2]).toBe(0);
|
||||
expect(samples[3]).toBe(0);
|
||||
expect(samples[5]).toBe(0.5);
|
||||
expect(samples[6]).toBeLessThan(0.95);
|
||||
expect(samples[6]).toBeGreaterThan(0.9);
|
||||
expect(samples[7]).toBe(0);
|
||||
expect(samples[8]).toBeLessThanOrEqual(0.986);
|
||||
});
|
||||
|
||||
it("runs the offline mix limiter for every rendered chunk", async () => {
|
||||
const processor = new AudioProcessor() as unknown as OfflineRenderTestHarness;
|
||||
const renderedSamples = new Float32Array([1.4]);
|
||||
const renderedBuffer = fakeAudioBuffer([renderedSamples]);
|
||||
const originalOfflineAudioContext = globalThis.OfflineAudioContext;
|
||||
(
|
||||
globalThis as unknown as { OfflineAudioContext: typeof OfflineAudioContext }
|
||||
).OfflineAudioContext = class {
|
||||
constructor() {}
|
||||
|
||||
startRendering() {
|
||||
return Promise.resolve(renderedBuffer);
|
||||
}
|
||||
} as unknown as typeof OfflineAudioContext;
|
||||
|
||||
try {
|
||||
let observedPeak = Number.POSITIVE_INFINITY;
|
||||
await processor.renderChunked(
|
||||
{
|
||||
mainBufferEntry: null,
|
||||
companionEntries: [],
|
||||
regionEntries: [],
|
||||
mutedSourceOutputRangesSec: [],
|
||||
slices: [],
|
||||
outputDurationMs: 100,
|
||||
numChannels: 1,
|
||||
},
|
||||
0.1,
|
||||
async (rendered) => {
|
||||
observedPeak = rendered.getChannelData(0)[0] ?? 0;
|
||||
},
|
||||
);
|
||||
|
||||
expect(observedPeak).toBeLessThanOrEqual(0.986);
|
||||
} finally {
|
||||
(
|
||||
globalThis as unknown as { OfflineAudioContext: typeof OfflineAudioContext }
|
||||
).OfflineAudioContext = originalOfflineAudioContext;
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,20 +1,17 @@
|
||||
import { WebDemuxer } from "web-demuxer";
|
||||
import { SOURCE_AUDIO_NORMALIZE_GAIN } from "@/components/video-editor/audio/audioTypes";
|
||||
import type {
|
||||
AudioRegion,
|
||||
ClipRegion,
|
||||
SpeedRegion,
|
||||
SourceAudioTrackSettings,
|
||||
SpeedRegion,
|
||||
TrimRegion,
|
||||
} from "@/components/video-editor/types";
|
||||
import {
|
||||
buildResolvedAudioPlan,
|
||||
SourceTrackId,
|
||||
} from "@/lib/exporter/audioRoutingEngine";
|
||||
import { buildResolvedAudioPlan, SourceTrackId } from "@/lib/exporter/audioRoutingEngine";
|
||||
import { estimateCompanionAudioStartDelaySeconds } from "@/lib/mediaTiming";
|
||||
import { resolveMediaElementSource } from "./localMediaSource";
|
||||
import type { VideoMuxer } from "./muxer";
|
||||
import { resolveSourceTrackRoutingPolicy } from "./sourceTrackRoutingPolicy";
|
||||
import { SOURCE_AUDIO_NORMALIZE_GAIN } from "@/components/video-editor/audio/audioTypes";
|
||||
|
||||
const AUDIO_BITRATE = 128_000;
|
||||
const DECODE_BACKPRESSURE_LIMIT = 20;
|
||||
@@ -24,6 +21,44 @@ const MP4_AUDIO_CODEC = "mp4a.40.2";
|
||||
const OFFLINE_AUDIO_SAMPLE_RATE = 48_000;
|
||||
const OFFLINE_ENCODE_CHUNK_FRAMES = 1024;
|
||||
const OFFLINE_CHUNK_DURATION_SEC = 30;
|
||||
const OFFLINE_MIX_SOFT_LIMITER_THRESHOLD = 0.9;
|
||||
const OFFLINE_MIX_SOFT_LIMITER_CEILING = 0.985;
|
||||
|
||||
function softLimitSample(sample: number): number {
|
||||
const magnitude = Math.abs(sample);
|
||||
if (magnitude <= OFFLINE_MIX_SOFT_LIMITER_THRESHOLD) {
|
||||
return sample;
|
||||
}
|
||||
|
||||
const sign = sample < 0 ? -1 : 1;
|
||||
const kneeRange = 1 - OFFLINE_MIX_SOFT_LIMITER_THRESHOLD;
|
||||
const limitedMagnitude =
|
||||
OFFLINE_MIX_SOFT_LIMITER_THRESHOLD +
|
||||
kneeRange * Math.tanh((magnitude - OFFLINE_MIX_SOFT_LIMITER_THRESHOLD) / kneeRange);
|
||||
return sign * Math.min(OFFLINE_MIX_SOFT_LIMITER_CEILING, limitedMagnitude);
|
||||
}
|
||||
|
||||
export function softLimitOfflineMixPeaksInPlace(buffer: AudioBuffer): boolean {
|
||||
let changed = false;
|
||||
for (let channel = 0; channel < buffer.numberOfChannels; channel += 1) {
|
||||
const data = buffer.getChannelData(channel);
|
||||
for (let index = 0; index < data.length; index += 1) {
|
||||
const sample = data[index];
|
||||
if (!Number.isFinite(sample)) {
|
||||
data[index] = 0;
|
||||
changed = true;
|
||||
continue;
|
||||
}
|
||||
|
||||
const limited = softLimitSample(sample);
|
||||
if (limited !== sample) {
|
||||
data[index] = limited;
|
||||
changed = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
return changed;
|
||||
}
|
||||
|
||||
function resolveSourceTrackGain(
|
||||
sourceAudioTrackSettings: SourceAudioTrackSettings | undefined,
|
||||
@@ -698,8 +733,11 @@ export class AudioProcessor {
|
||||
if (this.cancelled) throw new Error("Export cancelled");
|
||||
|
||||
// Decode companion / sidecar audio files
|
||||
const companionEntries: Array<{ buffer: AudioBuffer; startDelaySec: number; gain: number }> =
|
||||
[];
|
||||
const companionEntries: Array<{
|
||||
buffer: AudioBuffer;
|
||||
startDelaySec: number;
|
||||
gain: number;
|
||||
}> = [];
|
||||
const refDuration =
|
||||
mainBuffer?.duration ??
|
||||
(resolvedPlan.playbackPaths.length > 0 ? await this.getMediaDurationSec(videoUrl) : 0);
|
||||
@@ -947,6 +985,7 @@ export class AudioProcessor {
|
||||
|
||||
const rendered = await offlineCtx.startRendering();
|
||||
if (this.cancelled) break;
|
||||
softLimitOfflineMixPeaksInPlace(rendered);
|
||||
|
||||
await onChunk(rendered, outputOffsetSec, i);
|
||||
|
||||
@@ -1459,7 +1498,9 @@ export class AudioProcessor {
|
||||
{
|
||||
startSec: localOutputStartSec + chunkOutputStartSec,
|
||||
endSec:
|
||||
localOutputStartSec + chunkOutputStartSec + effectiveSourceDurationSec / slice.speed,
|
||||
localOutputStartSec +
|
||||
chunkOutputStartSec +
|
||||
effectiveSourceDurationSec / slice.speed,
|
||||
},
|
||||
];
|
||||
for (const mutedRange of mutedOutputRangesSec) {
|
||||
@@ -1491,7 +1532,8 @@ export class AudioProcessor {
|
||||
|
||||
const sourceOffsetSec =
|
||||
effectiveBufferStartSec +
|
||||
(audibleRange.startSec - (localOutputStartSec + chunkOutputStartSec)) * slice.speed;
|
||||
(audibleRange.startSec - (localOutputStartSec + chunkOutputStartSec)) *
|
||||
slice.speed;
|
||||
const localStartSec = audibleRange.startSec - chunkOutputStartSec;
|
||||
const sourceDurationSec = audibleDurationSec * slice.speed;
|
||||
|
||||
@@ -1543,7 +1585,9 @@ export class AudioProcessor {
|
||||
if (copyLength > 0) {
|
||||
for (let c = 0; c < channels; c++) {
|
||||
outBuffer.copyToChannel(
|
||||
originalBuffer.getChannelData(c).subarray(startSample, startSample + copyLength),
|
||||
originalBuffer
|
||||
.getChannelData(c)
|
||||
.subarray(startSample, startSample + copyLength),
|
||||
c,
|
||||
);
|
||||
}
|
||||
@@ -1559,7 +1603,7 @@ export class AudioProcessor {
|
||||
|
||||
const workStartIn = Math.max(0, startSample - paddingInSamples);
|
||||
const workEndIn = Math.min(originalBuffer.length, endSample + paddingInSamples);
|
||||
|
||||
|
||||
const actualPaddingInStart = startSample - workStartIn;
|
||||
// We expect the output offset for the requested start to be roughly:
|
||||
const actualPaddingOutStart = Math.floor(actualPaddingInStart / speed);
|
||||
@@ -1572,8 +1616,12 @@ export class AudioProcessor {
|
||||
const workOutSamples = Math.floor((workEndIn - workStartIn) / speed) + windowSize * 2;
|
||||
const workOutBuffer = ctx.createBuffer(channels, workOutSamples, sampleRate);
|
||||
|
||||
const inDataByChannel = Array.from({ length: channels }, (_, c) => originalBuffer.getChannelData(c));
|
||||
const workOutDataByChannel = Array.from({ length: channels }, (_, c) => workOutBuffer.getChannelData(c));
|
||||
const inDataByChannel = Array.from({ length: channels }, (_, c) =>
|
||||
originalBuffer.getChannelData(c),
|
||||
);
|
||||
const workOutDataByChannel = Array.from({ length: channels }, (_, c) =>
|
||||
workOutBuffer.getChannelData(c),
|
||||
);
|
||||
|
||||
const window = new Float32Array(windowSize);
|
||||
for (let i = 0; i < windowSize; i++) {
|
||||
@@ -1587,7 +1635,8 @@ export class AudioProcessor {
|
||||
for (let i = 0; i < windowSize; i++) {
|
||||
if (inOffset + i < workEndIn && outOffset + i < workOutSamples) {
|
||||
for (let c = 0; c < channels; c++) {
|
||||
workOutDataByChannel[c][outOffset + i] += inDataByChannel[c][inOffset + i] * window[i];
|
||||
workOutDataByChannel[c][outOffset + i] +=
|
||||
inDataByChannel[c][inOffset + i] * window[i];
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1609,7 +1658,9 @@ export class AudioProcessor {
|
||||
for (let i = 0; i < hopOut; i += 4) {
|
||||
if (outOffset + i < workOutSamples && testOffset + i < workEndIn) {
|
||||
for (let c = 0; c < channels; c++) {
|
||||
corr += workOutDataByChannel[c][outOffset + i] * inDataByChannel[c][testOffset + i];
|
||||
corr +=
|
||||
workOutDataByChannel[c][outOffset + i] *
|
||||
inDataByChannel[c][testOffset + i];
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1624,7 +1675,8 @@ export class AudioProcessor {
|
||||
for (let i = 0; i < windowSize; i++) {
|
||||
if (bestOffset + i < workEndIn && outOffset + i < workOutSamples) {
|
||||
for (let c = 0; c < channels; c++) {
|
||||
workOutDataByChannel[c][outOffset + i] += inDataByChannel[c][bestOffset + i] * window[i];
|
||||
workOutDataByChannel[c][outOffset + i] +=
|
||||
inDataByChannel[c][bestOffset + i] * window[i];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user