mirror of
https://github.com/webadderallorg/Recordly.git
synced 2026-09-24 23:05:49 +00:00
audio encoding works with exporting.
This commit is contained in:
@@ -4405,6 +4405,7 @@ export default function VideoEditor() {
|
||||
sourceAudioFallbackPaths: audio.sourceAudioFallbackPaths,
|
||||
sourceAudioFallbackStartDelayMsByPath:
|
||||
audio.sourceAudioFallbackStartDelayMsByPath,
|
||||
sourceAudioTrackSettings: audio.activeSourceAudioTrackSettings,
|
||||
previewWidth,
|
||||
previewHeight,
|
||||
onProgress: (progress: ExportProgress) => {
|
||||
@@ -6064,7 +6065,7 @@ export default function VideoEditor() {
|
||||
}
|
||||
cursorSway={cursorSway}
|
||||
volume={
|
||||
audio.isCurrentClipMuted
|
||||
audio.shouldMutePreviewVideo || audio.isCurrentClipMuted
|
||||
? 0
|
||||
: Math.max(
|
||||
0,
|
||||
|
||||
@@ -10,9 +10,6 @@ import type { AudioRegion, SpeedRegion } from "../types";
|
||||
|
||||
const SOURCE_AUDIO_PREVIEW_PLAYING_SEEK_DRIFT_SECONDS = 0.18;
|
||||
const SOURCE_AUDIO_PREVIEW_PAUSED_SEEK_DRIFT_SECONDS = 0.01;
|
||||
const SOURCE_AUDIO_PREVIEW_RATE_TOLERANCE_SECONDS = 0.08;
|
||||
const SOURCE_AUDIO_PREVIEW_RATE_CORRECTION_WINDOW_SECONDS = 8;
|
||||
const SOURCE_AUDIO_PREVIEW_MAX_RATE_ADJUSTMENT = 0.015;
|
||||
|
||||
interface UseAudioPreviewSyncParams {
|
||||
audioRegions: AudioRegion[];
|
||||
@@ -47,10 +44,43 @@ export function useAudioPreviewSync({
|
||||
const audioElementRevokersRef = useRef<Map<string, () => void>>(new Map());
|
||||
const audioElementResourcesRef = useRef<Map<string, string>>(new Map());
|
||||
const sourceAudioElementsRef = useRef<Map<string, HTMLAudioElement>>(new Map());
|
||||
const sourceAudioMediaNodesRef = useRef<Map<string, MediaElementAudioSourceNode>>(new Map());
|
||||
const sourceAudioGainNodesRef = useRef<Map<string, GainNode>>(new Map());
|
||||
const sourceAudioElementRevokersRef = useRef<Map<string, () => void>>(new Map());
|
||||
const sourceAudioElementResourcesRef = useRef<Map<string, string>>(new Map());
|
||||
const sourceAudioContextRef = useRef<AudioContext | null>(null);
|
||||
const sourceAudioMasterGainRef = useRef<GainNode | null>(null);
|
||||
const sourceAudioResumePromiseRef = useRef<Promise<void> | null>(null);
|
||||
const lastSourceAudioSyncTimeRef = useRef<number | null>(null);
|
||||
|
||||
const ensureSourceAudioContext = () => {
|
||||
if (!sourceAudioContextRef.current) {
|
||||
const context = new AudioContext({ latencyHint: "interactive" });
|
||||
const masterGain = context.createGain();
|
||||
masterGain.gain.value = 1;
|
||||
masterGain.connect(context.destination);
|
||||
sourceAudioContextRef.current = context;
|
||||
sourceAudioMasterGainRef.current = masterGain;
|
||||
}
|
||||
return sourceAudioContextRef.current;
|
||||
};
|
||||
|
||||
const ensureSourceAudioRunning = () => {
|
||||
const context = ensureSourceAudioContext();
|
||||
if (context.state === "running") {
|
||||
return Promise.resolve();
|
||||
}
|
||||
if (!sourceAudioResumePromiseRef.current) {
|
||||
sourceAudioResumePromiseRef.current = context
|
||||
.resume()
|
||||
.catch(() => undefined)
|
||||
.finally(() => {
|
||||
sourceAudioResumePromiseRef.current = null;
|
||||
});
|
||||
}
|
||||
return sourceAudioResumePromiseRef.current;
|
||||
};
|
||||
|
||||
useEffect(() => {
|
||||
let cancelled = false;
|
||||
const existing = audioElementsRef.current;
|
||||
@@ -117,6 +147,10 @@ export function useAudioPreviewSync({
|
||||
if (!currentIds.has(id)) {
|
||||
audio.pause();
|
||||
audio.src = "";
|
||||
sourceAudioMediaNodesRef.current.get(id)?.disconnect();
|
||||
sourceAudioMediaNodesRef.current.delete(id);
|
||||
sourceAudioGainNodesRef.current.get(id)?.disconnect();
|
||||
sourceAudioGainNodesRef.current.delete(id);
|
||||
sourceAudioElementRevokersRef.current.get(id)?.();
|
||||
sourceAudioElementRevokersRef.current.delete(id);
|
||||
sourceAudioElementResourcesRef.current.delete(id);
|
||||
@@ -129,10 +163,27 @@ export function useAudioPreviewSync({
|
||||
if (!audio) {
|
||||
audio = new Audio();
|
||||
audio.preload = "auto";
|
||||
audio.crossOrigin = "anonymous";
|
||||
existing.set(audioPath, audio);
|
||||
}
|
||||
audio.volume = 1;
|
||||
audio.dataset.sourceAudioPath = audioPath;
|
||||
|
||||
const context = ensureSourceAudioContext();
|
||||
const masterGain = sourceAudioMasterGainRef.current;
|
||||
if (context && masterGain && !sourceAudioMediaNodesRef.current.has(audioPath)) {
|
||||
try {
|
||||
const mediaNode = context.createMediaElementSource(audio);
|
||||
const trackGainNode = context.createGain();
|
||||
mediaNode.connect(trackGainNode);
|
||||
trackGainNode.connect(masterGain);
|
||||
sourceAudioMediaNodesRef.current.set(audioPath, mediaNode);
|
||||
sourceAudioGainNodesRef.current.set(audioPath, trackGainNode);
|
||||
} catch (error) {
|
||||
onSourceFallbackLoadError(error);
|
||||
}
|
||||
}
|
||||
|
||||
if (sourceAudioElementResourcesRef.current.get(audioPath) !== audioPath) {
|
||||
audio.pause();
|
||||
audio.src = "";
|
||||
@@ -174,9 +225,16 @@ export function useAudioPreviewSync({
|
||||
})();
|
||||
}
|
||||
|
||||
audio.volume = isCurrentClipMuted
|
||||
const trackGainNode = sourceAudioGainNodesRef.current.get(audioPath);
|
||||
if (trackGainNode) {
|
||||
trackGainNode.gain.value = Math.max(0, Math.min(2, getSourceTrackPreviewGain(audioPath)));
|
||||
}
|
||||
}
|
||||
|
||||
if (sourceAudioMasterGainRef.current) {
|
||||
sourceAudioMasterGainRef.current.gain.value = isCurrentClipMuted
|
||||
? 0
|
||||
: Math.max(0, Math.min(1, previewVolume * getSourceTrackPreviewGain(audioPath)));
|
||||
: Math.max(0, Math.min(1, previewVolume));
|
||||
}
|
||||
|
||||
if (previewSourceAudioFallbackPaths.length === 0) {
|
||||
@@ -210,12 +268,30 @@ export function useAudioPreviewSync({
|
||||
audio.pause();
|
||||
audio.src = "";
|
||||
}
|
||||
for (const node of sourceAudioMediaNodesRef.current.values()) {
|
||||
node.disconnect();
|
||||
}
|
||||
for (const node of sourceAudioGainNodesRef.current.values()) {
|
||||
node.disconnect();
|
||||
}
|
||||
for (const revoke of sourceAudioElementRevokersRef.current.values()) {
|
||||
revoke();
|
||||
}
|
||||
sourceAudioElementsRef.current.clear();
|
||||
sourceAudioMediaNodesRef.current.clear();
|
||||
sourceAudioGainNodesRef.current.clear();
|
||||
sourceAudioElementRevokersRef.current.clear();
|
||||
sourceAudioElementResourcesRef.current.clear();
|
||||
if (sourceAudioMasterGainRef.current) {
|
||||
sourceAudioMasterGainRef.current.disconnect();
|
||||
sourceAudioMasterGainRef.current = null;
|
||||
}
|
||||
const context = sourceAudioContextRef.current;
|
||||
sourceAudioContextRef.current = null;
|
||||
sourceAudioResumePromiseRef.current = null;
|
||||
if (context) {
|
||||
void context.close();
|
||||
}
|
||||
lastSourceAudioSyncTimeRef.current = null;
|
||||
};
|
||||
}, []);
|
||||
@@ -272,12 +348,18 @@ export function useAudioPreviewSync({
|
||||
const driftThreshold = isPlaying
|
||||
? SOURCE_AUDIO_PREVIEW_PLAYING_SEEK_DRIFT_SECONDS
|
||||
: SOURCE_AUDIO_PREVIEW_PAUSED_SEEK_DRIFT_SECONDS;
|
||||
if (sourceAudioMasterGainRef.current) {
|
||||
sourceAudioMasterGainRef.current.gain.value = isCurrentClipMuted
|
||||
? 0
|
||||
: Math.max(0, Math.min(1, previewVolume));
|
||||
}
|
||||
|
||||
for (const audio of sourceAudioElementsRef.current.values()) {
|
||||
const sourceAudioPath = audio.dataset.sourceAudioPath ?? "";
|
||||
audio.volume = isCurrentClipMuted
|
||||
? 0
|
||||
: Math.max(0, Math.min(1, previewVolume * getSourceTrackPreviewGain(sourceAudioPath)));
|
||||
const trackGainNode = sourceAudioGainNodesRef.current.get(sourceAudioPath);
|
||||
if (trackGainNode) {
|
||||
trackGainNode.gain.value = Math.max(0, Math.min(2, getSourceTrackPreviewGain(sourceAudioPath)));
|
||||
}
|
||||
|
||||
enablePitchPreservingPlayback(audio);
|
||||
const audioDuration = Number.isFinite(audio.duration) ? audio.duration : null;
|
||||
@@ -298,7 +380,11 @@ export function useAudioPreviewSync({
|
||||
const beforeAudioStart = currentTime + 0.001 < startDelaySeconds;
|
||||
const targetTime = clampMediaTimeToDuration(currentTime - startDelaySeconds, audioDuration);
|
||||
|
||||
if (timelineJumped || Math.abs(audio.currentTime - targetTime) > driftThreshold) {
|
||||
const shouldSeek =
|
||||
timelineJumped ||
|
||||
(!isPlaying && Math.abs(audio.currentTime - targetTime) > driftThreshold) ||
|
||||
(isPlaying && Math.abs(audio.currentTime - targetTime) > 0.9);
|
||||
if (shouldSeek) {
|
||||
try {
|
||||
audio.currentTime = targetTime;
|
||||
} catch {
|
||||
@@ -306,21 +392,18 @@ export function useAudioPreviewSync({
|
||||
}
|
||||
}
|
||||
|
||||
const syncedPlaybackRate = getMediaSyncPlaybackRate({
|
||||
basePlaybackRate: targetPlaybackRate,
|
||||
currentTime: audio.currentTime,
|
||||
targetTime,
|
||||
toleranceSeconds: SOURCE_AUDIO_PREVIEW_RATE_TOLERANCE_SECONDS,
|
||||
correctionWindowSeconds: SOURCE_AUDIO_PREVIEW_RATE_CORRECTION_WINDOW_SECONDS,
|
||||
maxAdjustment: SOURCE_AUDIO_PREVIEW_MAX_RATE_ADJUSTMENT,
|
||||
});
|
||||
// KISS for companion source tracks: fixed playback rate avoids audible flutter/stutter
|
||||
// from continuous micro-corrections on system audio.
|
||||
const syncedPlaybackRate = targetPlaybackRate;
|
||||
if (Math.abs(audio.playbackRate - syncedPlaybackRate) > 0.001) {
|
||||
audio.playbackRate = syncedPlaybackRate;
|
||||
}
|
||||
|
||||
const atEnd = audioDuration !== null && targetTime >= audioDuration;
|
||||
if (isPlaying && !beforeAudioStart && !atEnd) {
|
||||
audio.play().catch(() => undefined);
|
||||
void ensureSourceAudioRunning().then(() => {
|
||||
audio.play().catch(() => undefined);
|
||||
});
|
||||
} else if (!audio.paused) {
|
||||
audio.pause();
|
||||
}
|
||||
@@ -338,4 +421,17 @@ export function useAudioPreviewSync({
|
||||
previewSourceAudioFallbackPaths,
|
||||
sourceAudioFallbackStartDelayMsByPath,
|
||||
]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!isPlaying || previewSourceAudioFallbackPaths.length === 0) {
|
||||
return;
|
||||
}
|
||||
void ensureSourceAudioRunning().then(() => {
|
||||
for (const audio of sourceAudioElementsRef.current.values()) {
|
||||
if (audio.paused) {
|
||||
audio.play().catch(() => undefined);
|
||||
}
|
||||
}
|
||||
});
|
||||
}, [isPlaying, previewSourceAudioFallbackPaths]);
|
||||
}
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import React, { useMemo } from "react";
|
||||
import { resolveSourceAudioFallbackPaths } from "@/lib/exporter/sourceAudioFallback";
|
||||
import { resolveSourceTrackRoutingPolicy } from "@/lib/exporter/sourceTrackRoutingPolicy";
|
||||
import type {
|
||||
AudioRegion,
|
||||
ClipRegion,
|
||||
@@ -78,16 +78,12 @@ export function useVideoEditorAudio({
|
||||
summarizeErrorMessage,
|
||||
});
|
||||
|
||||
const { hasEmbeddedSourceAudio, externalAudioPaths: previewSourceAudioFallbackPaths } = useMemo(
|
||||
() => resolveSourceAudioFallbackPaths(currentSourcePath, sourceAudioFallbackPaths),
|
||||
const sourceTrackRoutingPolicy = useMemo(
|
||||
() => resolveSourceTrackRoutingPolicy(currentSourcePath, sourceAudioFallbackPaths),
|
||||
[currentSourcePath, sourceAudioFallbackPaths],
|
||||
);
|
||||
const hasSystemCompanionPreviewTrack = previewSourceAudioFallbackPaths.some((audioPath) =>
|
||||
audioPath.toLowerCase().includes(".system."),
|
||||
);
|
||||
const shouldMutePreviewVideo =
|
||||
previewSourceAudioFallbackPaths.length > 0 &&
|
||||
(!hasEmbeddedSourceAudio || hasSystemCompanionPreviewTrack);
|
||||
const previewSourceAudioFallbackPaths = sourceTrackRoutingPolicy.playbackPaths;
|
||||
const shouldMutePreviewVideo = sourceTrackRoutingPolicy.muteEmbeddedPreview;
|
||||
|
||||
const activeClipIdAtCurrentTime = useMemo(
|
||||
() => getActiveClipIdAtSourceTime(currentTime, clipRegions),
|
||||
|
||||
@@ -3,12 +3,16 @@ import type {
|
||||
AudioRegion,
|
||||
ClipRegion,
|
||||
SpeedRegion,
|
||||
SourceAudioTrackSettings,
|
||||
TrimRegion,
|
||||
} from "@/components/video-editor/types";
|
||||
import { estimateCompanionAudioStartDelaySeconds } from "@/lib/mediaTiming";
|
||||
import { resolveMediaElementSource } from "./localMediaSource";
|
||||
import type { VideoMuxer } from "./muxer";
|
||||
import { resolveSourceAudioFallbackPaths } from "./sourceAudioFallback";
|
||||
import {
|
||||
getSourceTrackIdFromPath,
|
||||
resolveSourceTrackRoutingPolicy,
|
||||
} from "./sourceTrackRoutingPolicy";
|
||||
|
||||
const AUDIO_BITRATE = 128_000;
|
||||
const DECODE_BACKPRESSURE_LIMIT = 20;
|
||||
@@ -26,8 +30,8 @@ interface TimelineSlice {
|
||||
}
|
||||
|
||||
interface PreparedOfflineRender {
|
||||
mainBuffer: AudioBuffer | null;
|
||||
companionEntries: Array<{ buffer: AudioBuffer; startDelaySec: number }>;
|
||||
mainBufferEntry: { buffer: AudioBuffer; gain: number } | null;
|
||||
companionEntries: Array<{ buffer: AudioBuffer; startDelaySec: number; gain: number }>;
|
||||
regionEntries: Array<{ buffer: AudioBuffer; region: AudioRegion }>;
|
||||
slices: TimelineSlice[];
|
||||
outputDurationMs: number;
|
||||
@@ -144,6 +148,7 @@ export class AudioProcessor {
|
||||
audioRegions?: AudioRegion[],
|
||||
sourceAudioFallbackPaths?: string[],
|
||||
sourceAudioFallbackStartDelayMsByPath?: Record<string, number>,
|
||||
sourceAudioTrackSettings?: SourceAudioTrackSettings,
|
||||
): Promise<void> {
|
||||
const sortedTrims = trimRegions
|
||||
? [...trimRegions].sort((a, b) => a.startMs - b.startMs)
|
||||
@@ -161,16 +166,16 @@ export class AudioProcessor {
|
||||
(audioPath) => typeof audioPath === "string" && audioPath.trim().length > 0,
|
||||
)
|
||||
: [];
|
||||
const { hasEmbeddedSourceAudio, externalAudioPaths } = resolveSourceAudioFallbackPaths(
|
||||
const routingPolicy = resolveSourceTrackRoutingPolicy(
|
||||
videoUrl,
|
||||
sortedSourceAudioFallbackPaths,
|
||||
);
|
||||
const hasTimedCompanionAudio = externalAudioPaths.some(
|
||||
const hasTimedCompanionAudio = routingPolicy.playbackPaths.some(
|
||||
(audioPath) => (sourceAudioFallbackStartDelayMsByPath?.[audioPath] ?? 0) > 0,
|
||||
);
|
||||
const needsSourceAudioMixing =
|
||||
externalAudioPaths.length > 1 ||
|
||||
(hasEmbeddedSourceAudio && externalAudioPaths.length > 0) ||
|
||||
routingPolicy.playbackPaths.length > 1 ||
|
||||
(routingPolicy.hasEmbeddedSourceAudio && routingPolicy.playbackPaths.length > 0) ||
|
||||
hasTimedCompanionAudio;
|
||||
|
||||
// When speed edits, audio regions, or multiple audio sources need mixing, use offline AudioContext pipeline.
|
||||
@@ -186,14 +191,15 @@ export class AudioProcessor {
|
||||
sortedAudioRegions,
|
||||
sortedSourceAudioFallbackPaths,
|
||||
sourceAudioFallbackStartDelayMsByPath,
|
||||
sourceAudioTrackSettings,
|
||||
muxer,
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
// Single sidecar audio with no speed/audio edits: demux directly (skips slow real-time rendering).
|
||||
if (!hasEmbeddedSourceAudio && externalAudioPaths.length === 1) {
|
||||
const sidecarDemuxer = await this.loadAudioFileDemuxer(externalAudioPaths[0]);
|
||||
if (!routingPolicy.hasEmbeddedSourceAudio && routingPolicy.playbackPaths.length === 1) {
|
||||
const sidecarDemuxer = await this.loadAudioFileDemuxer(routingPolicy.playbackPaths[0]);
|
||||
if (sidecarDemuxer) {
|
||||
try {
|
||||
await this.processTrimOnlyAudio(sidecarDemuxer, muxer, sortedTrims);
|
||||
@@ -215,8 +221,9 @@ export class AudioProcessor {
|
||||
sortedTrims,
|
||||
[],
|
||||
[],
|
||||
externalAudioPaths,
|
||||
routingPolicy.playbackPaths,
|
||||
sourceAudioFallbackStartDelayMsByPath,
|
||||
sourceAudioTrackSettings,
|
||||
muxer,
|
||||
);
|
||||
return;
|
||||
@@ -263,6 +270,7 @@ export class AudioProcessor {
|
||||
audioRegions?: AudioRegion[],
|
||||
sourceAudioFallbackPaths?: string[],
|
||||
sourceAudioFallbackStartDelayMsByPath?: Record<string, number>,
|
||||
sourceAudioTrackSettings?: SourceAudioTrackSettings,
|
||||
): Promise<Blob> {
|
||||
const sortedTrims = trimRegions
|
||||
? [...trimRegions].sort((a, b) => a.startMs - b.startMs)
|
||||
@@ -288,6 +296,7 @@ export class AudioProcessor {
|
||||
sortedAudioRegions,
|
||||
sortedSourceAudioFallbackPaths,
|
||||
sourceAudioFallbackStartDelayMsByPath,
|
||||
sourceAudioTrackSettings,
|
||||
);
|
||||
return this.renderToWavBlobChunked(prepared);
|
||||
}
|
||||
@@ -563,6 +572,7 @@ export class AudioProcessor {
|
||||
audioRegions: AudioRegion[],
|
||||
sourceAudioFallbackPaths: string[],
|
||||
sourceAudioFallbackStartDelayMsByPath: Record<string, number> | undefined,
|
||||
sourceAudioTrackSettings: SourceAudioTrackSettings | undefined,
|
||||
muxer: VideoMuxer,
|
||||
): Promise<void> {
|
||||
const prepared = await this.prepareOfflineRender(
|
||||
@@ -572,6 +582,7 @@ export class AudioProcessor {
|
||||
audioRegions,
|
||||
sourceAudioFallbackPaths,
|
||||
sourceAudioFallbackStartDelayMsByPath,
|
||||
sourceAudioTrackSettings,
|
||||
);
|
||||
if (this.cancelled) return;
|
||||
await this.renderAndEncodeChunked(prepared, muxer);
|
||||
@@ -584,31 +595,45 @@ export class AudioProcessor {
|
||||
audioRegions: AudioRegion[],
|
||||
sourceAudioFallbackPaths: string[],
|
||||
sourceAudioFallbackStartDelayMsByPath?: Record<string, number>,
|
||||
sourceAudioTrackSettings?: SourceAudioTrackSettings,
|
||||
): Promise<PreparedOfflineRender> {
|
||||
if (this.cancelled) throw new Error("Export cancelled");
|
||||
this.onProgress?.(0);
|
||||
|
||||
const { externalAudioPaths } = resolveSourceAudioFallbackPaths(
|
||||
const routingPolicy = resolveSourceTrackRoutingPolicy(
|
||||
videoUrl,
|
||||
sourceAudioFallbackPaths,
|
||||
);
|
||||
|
||||
// Decode embedded source audio separately from companion sidecars.
|
||||
const mainBuffer = await this.decodeAudioFromUrl(videoUrl);
|
||||
const mainBuffer = routingPolicy.includeEmbeddedInExport
|
||||
? await this.decodeAudioFromUrl(videoUrl)
|
||||
: null;
|
||||
const mainBufferGainSettings =
|
||||
sourceAudioTrackSettings?.mixed ?? sourceAudioTrackSettings?.system ?? null;
|
||||
const mainBufferGain = mainBufferGainSettings
|
||||
? Math.max(0, Math.min(2, mainBufferGainSettings.volume))
|
||||
: 1;
|
||||
const mainBufferEntry = mainBuffer ? { buffer: mainBuffer, gain: mainBufferGain } : null;
|
||||
if (this.cancelled) throw new Error("Export cancelled");
|
||||
|
||||
// Decode companion / sidecar audio files
|
||||
const companionEntries: Array<{ buffer: AudioBuffer; startDelaySec: number }> = [];
|
||||
const companionEntries: Array<{ buffer: AudioBuffer; startDelaySec: number; gain: number }> =
|
||||
[];
|
||||
const refDuration =
|
||||
mainBuffer?.duration ??
|
||||
(externalAudioPaths.length > 0 ? await this.getMediaDurationSec(videoUrl) : 0);
|
||||
for (const audioPath of externalAudioPaths) {
|
||||
(routingPolicy.playbackPaths.length > 0 ? await this.getMediaDurationSec(videoUrl) : 0);
|
||||
for (const audioPath of routingPolicy.playbackPaths) {
|
||||
if (this.cancelled) throw new Error("Export cancelled");
|
||||
const buffer = await this.decodeAudioFromUrl(audioPath);
|
||||
if (!buffer) continue;
|
||||
|
||||
companionEntries.push({
|
||||
buffer,
|
||||
gain: Math.max(
|
||||
0,
|
||||
Math.min(2, sourceAudioTrackSettings?.[getSourceTrackIdFromPath(audioPath)]?.volume ?? 1),
|
||||
),
|
||||
startDelaySec: estimateCompanionAudioStartDelaySeconds(
|
||||
refDuration,
|
||||
buffer.duration,
|
||||
@@ -629,15 +654,15 @@ export class AudioProcessor {
|
||||
this.onProgress?.(0.2);
|
||||
|
||||
// Determine source duration for timeline calculation
|
||||
const primaryBuffer = mainBuffer ?? companionEntries[0]?.buffer ?? null;
|
||||
const primaryBuffer = mainBufferEntry?.buffer ?? companionEntries[0]?.buffer ?? null;
|
||||
if (!primaryBuffer && regionEntries.length === 0) {
|
||||
throw new Error("No decodable audio sources found");
|
||||
}
|
||||
|
||||
let sourceDurationSec: number;
|
||||
if (mainBuffer) {
|
||||
sourceDurationSec = mainBuffer.duration;
|
||||
} else if (externalAudioPaths.length > 0 || regionEntries.length > 0) {
|
||||
if (mainBufferEntry?.buffer) {
|
||||
sourceDurationSec = mainBufferEntry.buffer.duration;
|
||||
} else if (routingPolicy.playbackPaths.length > 0 || regionEntries.length > 0) {
|
||||
sourceDurationSec = await this.getMediaDurationSec(videoUrl);
|
||||
} else {
|
||||
sourceDurationSec = primaryBuffer?.duration ?? 0;
|
||||
@@ -661,7 +686,7 @@ export class AudioProcessor {
|
||||
const numChannels = Math.min(primaryBuffer?.numberOfChannels ?? 2, 2);
|
||||
|
||||
return {
|
||||
mainBuffer,
|
||||
mainBufferEntry,
|
||||
companionEntries,
|
||||
regionEntries,
|
||||
slices,
|
||||
@@ -788,12 +813,13 @@ export class AudioProcessor {
|
||||
);
|
||||
|
||||
// Schedule main audio
|
||||
if (prepared.mainBuffer) {
|
||||
if (prepared.mainBufferEntry) {
|
||||
this.scheduleBufferThroughTimeline(
|
||||
offlineCtx,
|
||||
prepared.mainBuffer,
|
||||
prepared.mainBufferEntry.buffer,
|
||||
slices,
|
||||
0,
|
||||
prepared.mainBufferEntry.gain,
|
||||
outputOffsetSec,
|
||||
chunkSec,
|
||||
);
|
||||
@@ -806,6 +832,7 @@ export class AudioProcessor {
|
||||
entry.buffer,
|
||||
slices,
|
||||
entry.startDelaySec,
|
||||
entry.gain,
|
||||
outputOffsetSec,
|
||||
chunkSec,
|
||||
);
|
||||
@@ -1265,6 +1292,7 @@ export class AudioProcessor {
|
||||
buffer: AudioBuffer,
|
||||
slices: TimelineSlice[],
|
||||
sourceStartDelaySec: number,
|
||||
gain = 1,
|
||||
chunkOutputStartSec = 0,
|
||||
chunkDurationSec = Number.POSITIVE_INFINITY,
|
||||
): void {
|
||||
@@ -1331,9 +1359,12 @@ export class AudioProcessor {
|
||||
}
|
||||
|
||||
const source = ctx.createBufferSource();
|
||||
const gainNode = ctx.createGain();
|
||||
gainNode.gain.value = Math.max(0, Math.min(2, gain));
|
||||
source.buffer = buffer;
|
||||
source.playbackRate.value = slice.speed;
|
||||
source.connect(ctx.destination);
|
||||
source.connect(gainNode);
|
||||
gainNode.connect(ctx.destination);
|
||||
|
||||
source.start(localOutputStartSec, effectiveBufferStartSec, effectiveSourceDurationSec);
|
||||
|
||||
|
||||
@@ -8,6 +8,7 @@ import type {
|
||||
CursorTelemetryPoint,
|
||||
Padding,
|
||||
SpeedRegion,
|
||||
SourceAudioTrackSettings,
|
||||
TrimRegion,
|
||||
WebcamOverlaySettings,
|
||||
ZoomMotionBlurTuning,
|
||||
@@ -137,6 +138,7 @@ interface VideoExporterConfig extends ExportConfig {
|
||||
audioRegions?: AudioRegion[];
|
||||
sourceAudioFallbackPaths?: string[];
|
||||
sourceAudioFallbackStartDelayMsByPath?: Record<string, number>;
|
||||
sourceAudioTrackSettings?: SourceAudioTrackSettings;
|
||||
previewWidth?: number;
|
||||
previewHeight?: number;
|
||||
onProgress?: (progress: ExportProgress) => void;
|
||||
@@ -752,6 +754,7 @@ export class ModernVideoExporter {
|
||||
this.config.audioRegions,
|
||||
this.config.sourceAudioFallbackPaths,
|
||||
this.config.sourceAudioFallbackStartDelayMsByPath,
|
||||
this.config.sourceAudioTrackSettings,
|
||||
),
|
||||
"audio processing",
|
||||
"audio",
|
||||
@@ -1805,6 +1808,7 @@ export class ModernVideoExporter {
|
||||
this.config.audioRegions,
|
||||
this.config.sourceAudioFallbackPaths,
|
||||
this.config.sourceAudioFallbackStartDelayMsByPath,
|
||||
this.config.sourceAudioTrackSettings,
|
||||
),
|
||||
description,
|
||||
"audio",
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { resolveSourceTrackRoutingPolicy } from "./sourceTrackRoutingPolicy";
|
||||
|
||||
describe("resolveSourceTrackRoutingPolicy", () => {
|
||||
it("prioritizes system+mic sidecars and mutes embedded preview", () => {
|
||||
const policy = resolveSourceTrackRoutingPolicy("/tmp/recording.mp4", [
|
||||
"/tmp/recording.mp4",
|
||||
"/tmp/recording.system.wav",
|
||||
"/tmp/recording.mic.wav",
|
||||
"/tmp/recording.mixed.wav",
|
||||
]);
|
||||
|
||||
expect(policy.playbackPaths).toEqual([
|
||||
"/tmp/recording.system.wav",
|
||||
"/tmp/recording.mic.wav",
|
||||
]);
|
||||
expect(policy.muteEmbeddedPreview).toBe(true);
|
||||
expect(policy.includeEmbeddedInExport).toBe(false);
|
||||
});
|
||||
|
||||
it("falls back to mixed when dedicated tracks are absent", () => {
|
||||
const policy = resolveSourceTrackRoutingPolicy("/tmp/recording.mp4", [
|
||||
"/tmp/recording.mixed.wav",
|
||||
]);
|
||||
|
||||
expect(policy.playbackPaths).toEqual(["/tmp/recording.mixed.wav"]);
|
||||
expect(policy.muteEmbeddedPreview).toBe(false);
|
||||
expect(policy.includeEmbeddedInExport).toBe(false);
|
||||
});
|
||||
|
||||
it("keeps embedded audio when only mic sidecar is present", () => {
|
||||
const policy = resolveSourceTrackRoutingPolicy("/tmp/recording.mp4", [
|
||||
"/tmp/recording.mp4",
|
||||
"/tmp/recording.mic.wav",
|
||||
]);
|
||||
|
||||
expect(policy.playbackPaths).toEqual(["/tmp/recording.mic.wav"]);
|
||||
expect(policy.muteEmbeddedPreview).toBe(true);
|
||||
expect(policy.includeEmbeddedInExport).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,50 @@
|
||||
import { resolveSourceAudioFallbackPaths } from "./sourceAudioFallback";
|
||||
|
||||
export type SourceTrackId = "mic" | "system" | "mixed";
|
||||
|
||||
export function getSourceTrackIdFromPath(audioPath: string): SourceTrackId {
|
||||
const normalized = audioPath.toLowerCase();
|
||||
if (normalized.includes(".mic.")) return "mic";
|
||||
if (normalized.includes(".system.")) return "system";
|
||||
return "mixed";
|
||||
}
|
||||
|
||||
export interface SourceTrackRoutingPolicy {
|
||||
hasEmbeddedSourceAudio: boolean;
|
||||
pathsByTrack: Partial<Record<SourceTrackId, string>>;
|
||||
playbackPaths: string[];
|
||||
muteEmbeddedPreview: boolean;
|
||||
includeEmbeddedInExport: boolean;
|
||||
}
|
||||
|
||||
export function resolveSourceTrackRoutingPolicy(
|
||||
videoResource: string | null | undefined,
|
||||
sourceAudioFallbackPaths: string[] | null | undefined,
|
||||
): SourceTrackRoutingPolicy {
|
||||
const { hasEmbeddedSourceAudio, externalAudioPaths } = resolveSourceAudioFallbackPaths(
|
||||
videoResource,
|
||||
sourceAudioFallbackPaths,
|
||||
);
|
||||
|
||||
const pathsByTrack: Partial<Record<SourceTrackId, string>> = {};
|
||||
for (const path of externalAudioPaths) {
|
||||
const trackId = getSourceTrackIdFromPath(path);
|
||||
if (!pathsByTrack[trackId]) {
|
||||
pathsByTrack[trackId] = path;
|
||||
}
|
||||
}
|
||||
|
||||
const hasDedicatedTracks = Boolean(pathsByTrack.system || pathsByTrack.mic);
|
||||
const playbackPaths: string[] = [];
|
||||
if (pathsByTrack.system) playbackPaths.push(pathsByTrack.system);
|
||||
if (pathsByTrack.mic) playbackPaths.push(pathsByTrack.mic);
|
||||
if (!hasDedicatedTracks && pathsByTrack.mixed) playbackPaths.push(pathsByTrack.mixed);
|
||||
|
||||
return {
|
||||
hasEmbeddedSourceAudio,
|
||||
pathsByTrack,
|
||||
playbackPaths,
|
||||
muteEmbeddedPreview: hasDedicatedTracks,
|
||||
includeEmbeddedInExport: !pathsByTrack.system && !pathsByTrack.mixed,
|
||||
};
|
||||
}
|
||||
@@ -8,6 +8,7 @@ import type {
|
||||
CursorTelemetryPoint,
|
||||
Padding,
|
||||
SpeedRegion,
|
||||
SourceAudioTrackSettings,
|
||||
TrimRegion,
|
||||
WebcamOverlaySettings,
|
||||
ZoomMotionBlurTuning,
|
||||
@@ -92,6 +93,7 @@ interface VideoExporterConfig extends ExportConfig {
|
||||
audioRegions?: AudioRegion[];
|
||||
sourceAudioFallbackPaths?: string[];
|
||||
sourceAudioFallbackStartDelayMsByPath?: Record<string, number>;
|
||||
sourceAudioTrackSettings?: SourceAudioTrackSettings;
|
||||
previewWidth?: number;
|
||||
previewHeight?: number;
|
||||
onProgress?: (progress: ExportProgress) => void;
|
||||
@@ -398,6 +400,7 @@ export class VideoExporter {
|
||||
this.config.audioRegions,
|
||||
this.config.sourceAudioFallbackPaths,
|
||||
this.config.sourceAudioFallbackStartDelayMsByPath,
|
||||
this.config.sourceAudioTrackSettings,
|
||||
),
|
||||
"audio processing",
|
||||
"audio",
|
||||
@@ -847,6 +850,7 @@ export class VideoExporter {
|
||||
this.config.audioRegions,
|
||||
this.config.sourceAudioFallbackPaths,
|
||||
this.config.sourceAudioFallbackStartDelayMsByPath,
|
||||
this.config.sourceAudioTrackSettings,
|
||||
),
|
||||
"native edited audio rendering",
|
||||
"audio",
|
||||
|
||||
Reference in New Issue
Block a user