Merge pull request #482 from ExtraBinoss/fix/audio-wrong-layer

Fix/audio wrong layer and audio playback
This commit is contained in:
ExtraBinoss
2026-05-09 22:29:31 +02:00
committed by GitHub
44 changed files with 2707 additions and 691 deletions
+30
View File
@@ -172,4 +172,34 @@ describe("local media path policy", () => {
expect(result.path).toBe(projectPath);
expect(result.project).toMatchObject({ videoPath });
});
it("approves editor audioRegions audioPath entries when loading a project", async () => {
const downloadsPath = path.join(tempRoot, "Downloads");
const videoPath = path.join(tempPath, "recording.mp4");
const audioPath = path.join(downloadsPath, "music.ogg");
const projectPath = path.join(tempPath, "recording.recordly");
await fs.mkdir(downloadsPath, { recursive: true });
await fs.writeFile(videoPath, "test-video");
await fs.writeFile(audioPath, "test-audio");
await fs.writeFile(
projectPath,
JSON.stringify({
version: 1,
videoPath,
editor: {
audioRegions: [
{ id: "a1", startMs: 0, endMs: 1000, audioPath, volume: 1 },
],
},
}),
"utf-8",
);
const { loadProjectFromPath, resolveApprovedLocalMediaPath } = await import("./manager");
const resolvedAudioPath = await fs.realpath(audioPath);
const result = await loadProjectFromPath(projectPath);
expect(result.success).toBe(true);
await expect(resolveApprovedLocalMediaPath(audioPath)).resolves.toBe(resolvedAudioPath);
});
});
+8
View File
@@ -428,6 +428,7 @@ export async function loadProjectFromPath(projectPath: string) {
const projectObj = project as Record<string, unknown>;
const editorObj = projectObj?.editor as Record<string, unknown> | undefined;
const audioTracks = editorObj?.audioTracks as { sourcePath?: unknown }[] | undefined;
const audioRegions = editorObj?.audioRegions as { audioPath?: unknown }[] | undefined;
const approvedProjectPaths: Array<string | null | undefined> = [
mediaSources.videoPath,
mediaSources.webcamPath,
@@ -439,6 +440,13 @@ export async function loadProjectFromPath(projectPath: string) {
}
}
}
if (Array.isArray(audioRegions)) {
for (const region of audioRegions) {
if (typeof region?.audioPath === "string") {
approvedProjectPaths.push(region.audioPath);
}
}
}
await replaceApprovedSessionLocalReadPaths(approvedProjectPaths);
await rememberRecentProject(normalizedPath);
+1 -1
View File
@@ -32,7 +32,7 @@ export function getBrowserMicSidecarFilters(profile?: string | null) {
return BROWSER_MIC_SIDECAR_FILTERS;
}
export const RECORDING_AUDIO_SIDECAR_DEBUG_ENV = "RECORDLY_KEEP_RECORDING_AUDIO_SIDECARS";
export const RECORDING_AUDIO_SIDECAR_DEBUG_ENV = "RECORDLY_KEEP_RECORDING_AUDIO_SIDECARS"; // not used yet, because we need to have seperate audio files for system and mic for each recording
export function shouldKeepRecordingAudioSidecars(env: NodeJS.ProcessEnv = process.env) {
const value = env[RECORDING_AUDIO_SIDECAR_DEBUG_ENV]?.trim().toLowerCase();
+5 -4
View File
@@ -499,20 +499,21 @@ export async function getCompanionAudioFallbackInfo(videoPath: string) {
let paths: string[];
if (await hasEmbeddedAudioStream(videoPath)) {
const microphoneCompanionPaths = Array.from(
const companionPaths = Array.from(
new Set(
companionCandidates.flatMap((candidate) =>
candidate.usablePaths.filter(
(companionPath) => companionPath === candidate.micPath,
(companionPath) =>
companionPath === candidate.micPath || companionPath === candidate.systemPath,
),
),
),
);
if (microphoneCompanionPaths.length === 0) {
if (companionPaths.length === 0) {
return { paths: [], startDelayMsByPath: {} };
}
paths = [videoPath, ...microphoneCompanionPaths];
paths = [videoPath, ...companionPaths];
} else {
paths = Array.from(
new Set(companionCandidates.flatMap((candidate) => candidate.usablePaths)),
-6
View File
@@ -277,12 +277,6 @@ export async function muxNativeMacRecordingWithAudio(
await moveFileWithOverwrite(mixedOutputPath, videoPath);
console.log("[mux] Successfully muxed audio into video:", videoPath);
for (const audioPath of [systemAudioPath, microphonePath]) {
if (audioPath) {
await fs.rm(audioPath, { force: true }).catch(() => undefined);
}
}
}
export function attachNativeCaptureLifecycle(process: ChildProcessWithoutNullStreams) {
+1 -26
View File
@@ -24,7 +24,6 @@ import {
import type { AudioSyncAdjustment } from "../types";
import { moveFileWithOverwrite } from "../utils";
import {
RECORDING_AUDIO_SIDECAR_DEBUG_ENV,
shouldKeepRecordingAudioSidecars,
WINDOWS_NATIVE_MIC_PRE_FILTERS,
} from "./audioFilters";
@@ -474,30 +473,6 @@ export async function muxNativeWindowsVideoWithAudio(
throw error;
}
if (keepAudioSidecars) {
console.log(
`[mux-win] Keeping native audio sidecars because ${RECORDING_AUDIO_SIDECAR_DEBUG_ENV} is enabled`,
);
return {
muxed: true,
videoDurationSeconds: videoDuration,
muxTimeoutMs,
audioInputs,
audio,
outputPath: videoPath,
keptAudioSidecars: true,
};
}
for (const audioPath of [systemAudioPath, micAudioPath]) {
if (audioPath) {
await Promise.all([
fs.rm(audioPath, { force: true }).catch(() => undefined),
fs.rm(`${audioPath}.json`, { force: true }).catch(() => undefined),
]);
}
}
return {
muxed: true,
videoDurationSeconds: videoDuration,
@@ -505,6 +480,6 @@ export async function muxNativeWindowsVideoWithAudio(
audioInputs,
audio,
outputPath: videoPath,
keptAudioSidecars: false,
keptAudioSidecars: true,
};
}
@@ -139,8 +139,9 @@ export function AnnotationSettingsPanel({
};
return (
<div className="flex-[2] min-w-0 bg-editor-panel border border-foreground/10 rounded-2xl p-4 flex flex-col shadow-xl h-full overflow-y-auto custom-scrollbar">
<div className="mb-6">
<div className="flex-[2] min-w-0 bg-editor-panel border border-foreground/10 rounded-2xl flex flex-col shadow-xl h-full overflow-hidden">
<div className="flex-1 min-h-0 p-4 overflow-y-auto custom-scrollbar">
<div className="mb-6">
<div className="flex items-center justify-between mb-4">
<span className="text-sm font-medium text-foreground">
{t("annotations.settings")}
@@ -772,16 +773,6 @@ export function AnnotationSettingsPanel({
</TabsContent>
</Tabs>
<Button
onClick={onDelete}
variant="destructive"
size="sm"
className="w-full gap-2 bg-red-500/10 text-red-400 border border-red-500/20 hover:bg-red-500/20 hover:border-red-500/30 transition-all mt-4"
>
<Trash2 className="w-4 h-4" />
{t("annotations.deleteAnnotation")}
</Button>
<div className="mt-6 p-3 bg-foreground/5 rounded-lg border border-foreground/5">
<div className="flex items-center gap-2 mb-2 text-muted-foreground">
<Info className="w-3.5 h-3.5" />
@@ -795,6 +786,18 @@ export function AnnotationSettingsPanel({
<li>{t("annotations.tipCycleBackward")}</li>
</ul>
</div>
</div>
</div>
<div className="flex-shrink-0 border-t border-foreground/10 bg-editor-panel p-4 pt-3">
<Button
onClick={onDelete}
variant="destructive"
size="sm"
className="w-full gap-2 bg-red-500/10 text-red-400 border border-red-500/20 hover:bg-red-500/20 hover:border-red-500/30 transition-all"
>
<Trash2 className="w-4 h-4" />
{t("annotations.deleteAnnotation")}
</Button>
</div>
</div>
);
+214 -56
View File
@@ -469,12 +469,21 @@ interface SettingsPanelProps {
selectedClipId?: string | null;
selectedClipSpeed?: number | null;
selectedClipMuted?: boolean | null;
selectedClipShowSourceAudio?: boolean | null;
hasClipSourceAudio?: boolean;
onClipSpeedChange?: (speed: number) => void;
onClipMutedChange?: (muted: boolean) => void;
onClipShowSourceAudioChange?: (show: boolean) => void;
sourceAudioTrackMeta?: Array<{ id: string; label: string }>;
sourceAudioTrackSettings?: Record<string, { volume: number; normalize: boolean }>;
onSourceAudioTrackVolumeChange?: (id: string, volume: number) => void;
onSourceAudioTrackNormalizeChange?: (id: string, normalize: boolean) => void;
onClipDelete?: (id: string) => void;
selectedAudioId?: string | null;
selectedAudioVolume?: number | null;
selectedAudioNormalize?: boolean | null;
onAudioVolumeChange?: (volume: number) => void;
onAudioNormalizeChange?: (normalize: boolean) => void;
onAudioDelete?: (id: string) => void;
shadowIntensity?: number;
onShadowChange?: (intensity: number) => void;
@@ -862,12 +871,21 @@ export function SettingsPanel({
selectedClipId,
selectedClipSpeed,
selectedClipMuted,
selectedClipShowSourceAudio = false,
hasClipSourceAudio = false,
onClipSpeedChange,
onClipMutedChange,
onClipShowSourceAudioChange,
sourceAudioTrackMeta = [],
sourceAudioTrackSettings = {},
onSourceAudioTrackVolumeChange,
onSourceAudioTrackNormalizeChange,
onClipDelete,
selectedAudioId,
selectedAudioVolume,
selectedAudioNormalize,
onAudioVolumeChange,
onAudioNormalizeChange,
onAudioDelete,
shadowIntensity = 0.67,
onShadowChange,
@@ -3035,6 +3053,45 @@ export function SettingsPanel({
</section>
);
const audioSectionContent = (
<section className="flex flex-col gap-3">
<div className="flex items-center justify-between gap-3">
<SectionLabel>{tSettings("audio.volumeTitle", "Audio")}</SectionLabel>
<button
type="button"
onClick={() => {
onAudioVolumeChange?.(1);
onAudioNormalizeChange?.(false);
}}
className="text-[10px] text-[#2563EB] transition-opacity hover:opacity-80"
>
{t("common.actions.reset", "Reset")}
</button>
</div>
<SliderControl
label={tSettings("audio.volume", "Volume")}
value={selectedAudioVolume ?? 1}
defaultValue={1}
min={0}
max={1}
step={0.01}
onChange={(v) => onAudioVolumeChange?.(v)}
formatValue={(v) => `${Math.round(v * 100)}%`}
parseInput={(text) => parseFloat(text.replace(/%$/, "")) / 100}
/>
<div className="flex items-center justify-between rounded-lg bg-foreground/[0.03] px-2.5 py-1.5">
<span className="text-[10px] text-muted-foreground">
{tSettings("audio.normalize", "Normalize")}
</span>
<Switch
checked={Boolean(selectedAudioNormalize)}
onCheckedChange={(v) => onAudioNormalizeChange?.(v)}
className="data-[state=checked]:bg-[#2563EB] scale-75"
/>
</div>
</section>
);
const clipSectionContent = (
<section className="flex flex-col gap-2">
<div className="flex items-center justify-between gap-3">
@@ -3045,16 +3102,7 @@ export function SettingsPanel({
</span>
)}
</div>
<div className="flex items-center justify-between rounded-lg bg-foreground/[0.03] px-2.5 py-1.5">
<span className="text-[10px] text-muted-foreground">
{tSettings("clip.muteAudio", "Mute Audio")}
</span>
<Switch
checked={selectedClipMuted ?? false}
onCheckedChange={(v) => onClipMutedChange?.(v)}
className="data-[state=checked]:bg-[#06b6d4] scale-75"
/>
</div>
<div className="flex items-center gap-3">
<SectionLabel>{tSettings("speed.label", "Speed")}</SectionLabel>
</div>
@@ -3095,19 +3143,100 @@ export function SettingsPanel({
);
})}
</div>
{selectedClipId && (
<Button
onClick={() => {
if (selectedClipId && onClipDelete) onClipDelete(selectedClipId);
}}
variant="destructive"
size="sm"
className="mt-1 h-8 w-full gap-2 border border-red-500/20 bg-red-500/10 text-xs text-red-400 transition-all hover:border-red-500/30 hover:bg-red-500/20"
>
<Trash2 className="h-3 w-3" />
{tSettings("clip.delete", "Delete Clip")}
</Button>
)}
<div className="mt-2 flex flex-col gap-2 border-t border-foreground/5 pt-3">
<SectionLabel>{tSettings("audio.title", "Audio")}</SectionLabel>
<div className="flex items-center justify-between rounded-lg bg-foreground/[0.03] px-2.5 py-1.5">
<div>
<span className="text-[10px] text-muted-foreground">
{tSettings("clip.mute", "Mute")}
</span>
<p className="text-[9px] text-muted-foreground/50 mt-0.5">
{selectedClipMuted
? tSettings("clip.mutedState", "Audio is muted")
: tSettings("clip.unmutedState", "Audio is playing")}
</p>
</div>
<Switch
checked={selectedClipMuted ?? false}
onCheckedChange={(v) => onClipMutedChange?.(v)}
className="data-[state=checked]:bg-[#06b6d4] scale-75"
/>
</div>
{hasClipSourceAudio && (
<div className="flex items-center justify-between rounded-lg bg-foreground/[0.03] px-2.5 py-1.5">
<span className="text-[10px] text-muted-foreground">
{tSettings("clip.separateClipFromAudio", "Separate clip from audio")}
</span>
<Switch
checked={selectedClipShowSourceAudio ?? false}
onCheckedChange={(v) => onClipShowSourceAudioChange?.(v)}
className="data-[state=checked]:bg-[#06b6d4] scale-75"
/>
</div>
)}
</div>
{selectedClipId &&
hasClipSourceAudio &&
sourceAudioTrackMeta.length > 0 && (
<div className="mt-1 flex flex-col gap-3">
{sourceAudioTrackMeta.map((track) => {
const settings = sourceAudioTrackSettings[track.id] ?? {
volume: 1,
normalize: false,
};
return (
<div
key={track.id}
className="rounded-lg border border-foreground/10 bg-foreground/[0.03] px-3 py-2"
>
<div className="mb-2 flex items-center justify-between">
<span className="text-[11px] font-medium text-foreground">
{track.label}
</span>
<button
type="button"
onClick={() => {
onSourceAudioTrackVolumeChange?.(track.id, 1);
onSourceAudioTrackNormalizeChange?.(track.id, false);
}}
className="text-[10px] text-[#2563EB] transition-opacity hover:opacity-80"
>
{t("common.actions.reset", "Reset")}
</button>
</div>
<div className="mb-2 flex items-center justify-between rounded-lg bg-foreground/[0.03] px-2.5 py-1.5">
<span className="text-[10px] text-muted-foreground">
{tSettings("audio.normalize", "Normalize")}
</span>
<Switch
checked={settings.normalize}
onCheckedChange={(v) =>
onSourceAudioTrackNormalizeChange?.(track.id, v)
}
className="data-[state=checked]:bg-[#06b6d4] scale-75"
/>
</div>
<SliderControl
label={tSettings("audio.volume", "Volume")}
value={settings.volume}
defaultValue={1}
min={0}
max={2}
step={0.01}
onChange={(v) => onSourceAudioTrackVolumeChange?.(track.id, v)}
formatValue={(v) => `${Math.round(v * 100)}%`}
parseInput={(text) =>
parseFloat(text.replace(/%$/, "")) / 100
}
/>
</div>
);
})}
</div>
)}
</section>
);
@@ -3120,6 +3249,8 @@ export function SettingsPanel({
return zoomItemSectionContent;
case "clip":
return clipSectionContent;
case "audio":
return audioSectionContent;
case "frame":
return sceneSectionContent;
case "crop":
@@ -3561,41 +3692,68 @@ export function SettingsPanel({
<div
className={cn(
"flex-shrink-0 border-t border-foreground/10 bg-editor-header p-4 pt-3",
!selectedAudioId && "hidden",
"flex-shrink-0 border-t border-foreground/10 bg-editor-panel p-4 pt-3",
(() => {
if (activeEffectSection === "clip" && selectedClipId) return false;
if (activeEffectSection === "zoom" && selectedZoomId) return false;
if (activeEffectSection === "audio" && selectedAudioId) return false;
if (selectedAnnotationId) return false; // Annotation editor handles its own but let's see
return true;
})() && "hidden",
)}
>
{selectedAudioId && (
<div>
<div className="mb-3 flex items-center justify-between">
<span className="text-sm font-medium text-foreground">
{tSettings("audio.volumeTitle", "Audio Volume")}
</span>
<span className="rounded-full bg-[#2563EB]/10 px-2 py-0.5 text-[10px] font-medium uppercase tracking-wider text-[#2563EB]">
{Math.round((selectedAudioVolume ?? 1) * 100)}%
</span>
</div>
<SliderControl
label={tSettings("audio.volume", "Volume")}
value={selectedAudioVolume ?? 1}
defaultValue={1}
min={0}
max={1}
step={0.01}
onChange={(v) => onAudioVolumeChange?.(v)}
formatValue={(v) => `${Math.round(v * 100)}%`}
parseInput={(text) => parseFloat(text.replace(/%$/, "")) / 100}
/>
<Button
onClick={() => selectedAudioId && onAudioDelete?.(selectedAudioId)}
variant="destructive"
size="sm"
className="mt-2 h-8 w-full gap-2 border border-red-500/20 bg-red-500/10 text-xs text-red-400 transition-all hover:border-red-500/30 hover:bg-red-500/20"
>
<Trash2 className="h-3 w-3" />
{tSettings("audio.deleteRegion", "Delete Audio")}
</Button>
</div>
{activeEffectSection === "clip" && selectedClipId && (
<Button
onClick={() => {
if (selectedClipId && onClipDelete) onClipDelete(selectedClipId);
}}
variant="destructive"
size="sm"
className="h-8 w-full gap-2 border border-red-500/20 bg-red-500/10 text-xs text-red-400 transition-all hover:border-red-500/30 hover:bg-red-500/20"
>
<Trash2 className="h-3 w-3" />
{tSettings("clip.delete", "Delete Clip")}
</Button>
)}
{activeEffectSection === "zoom" && selectedZoomId && (
<Button
onClick={() => {
if (selectedZoomId && onZoomDelete) onZoomDelete(selectedZoomId);
}}
variant="destructive"
size="sm"
className="h-8 w-full gap-2 border border-red-500/20 bg-red-500/10 text-xs text-red-400 transition-all hover:border-red-500/30 hover:bg-red-500/20"
>
<Trash2 className="h-3 w-3" />
{tSettings("zoom.deleteZoom", "Delete Zoom")}
</Button>
)}
{activeEffectSection === "audio" && selectedAudioId && (
<Button
onClick={() => {
if (selectedAudioId && onAudioDelete) onAudioDelete(selectedAudioId);
}}
variant="destructive"
size="sm"
className="h-8 w-full gap-2 border border-red-500/20 bg-red-500/10 text-xs text-red-400 transition-all hover:border-red-500/30 hover:bg-red-500/20"
>
<Trash2 className="h-3 w-3" />
{tSettings("audio.deleteRegion", "Delete Audio")}
</Button>
)}
{selectedAnnotationId && (
<Button
onClick={() => {
if (selectedAnnotationId && onAnnotationDelete)
onAnnotationDelete(selectedAnnotationId);
}}
variant="destructive"
size="sm"
className="h-8 w-full gap-2 border border-red-500/20 bg-red-500/10 text-xs text-red-400 transition-all hover:border-red-500/30 hover:bg-red-500/20"
>
<Trash2 className="h-3 w-3" />
{tSettings("annotation.delete", "Delete Annotation")}
</Button>
)}
</div>
</div>
+156 -394
View File
@@ -80,14 +80,6 @@ import {
canUseInMemoryExportSaveFallback,
describeBlockedInMemoryExportSave,
} from "@/lib/exporter/exportSavePolicy";
import { resolveMediaElementSource } from "@/lib/exporter/localMediaSource";
import { resolveSourceAudioFallbackPaths } from "@/lib/exporter/sourceAudioFallback";
import {
clampMediaTimeToDuration,
enablePitchPreservingPlayback,
estimateCompanionAudioStartDelaySeconds,
getMediaSyncPlaybackRate,
} from "@/lib/mediaTiming";
import { matchesShortcut } from "@/lib/shortcuts";
import { cn } from "@/lib/utils";
import {
@@ -147,6 +139,7 @@ import {
validateProjectData,
} from "./projectPersistence";
import { SettingsPanel } from "./SettingsPanel";
import { useVideoEditorAudio } from "./audio/useVideoEditorAudio";
import {
APP_HEADER_ICON_BUTTON_CLASS,
DiscordLinkButton,
@@ -156,6 +149,7 @@ import {
} from "./TutorialHelp";
import TimelineEditor, { type TimelineEditorHandle } from "./timeline/TimelineEditor";
import { normalizeCursorTelemetry } from "./timeline/zoomSuggestionUtils";
import type { SourceAudioTrackSettings } from "@/components/video-editor/audio/audioTypes";
import {
type AnnotationRegion,
type AudioRegion,
@@ -347,12 +341,6 @@ async function writeSmokeExportReport(
const SMOKE_EXPORT_READY_TIMEOUT_MS = 30_000;
const DEFAULT_MP4_EXPORT_FRAME_RATE: ExportMp4FrameRate = 30;
const SOURCE_AUDIO_FALLBACK_TOAST_ID = "source-audio-fallback-error";
const SOURCE_AUDIO_PREVIEW_PLAYING_SEEK_DRIFT_SECONDS = 0.18;
const SOURCE_AUDIO_PREVIEW_PAUSED_SEEK_DRIFT_SECONDS = 0.01;
const SOURCE_AUDIO_PREVIEW_RATE_TOLERANCE_SECONDS = 0.08;
const SOURCE_AUDIO_PREVIEW_RATE_CORRECTION_WINDOW_SECONDS = 8;
const SOURCE_AUDIO_PREVIEW_MAX_RATE_ADJUSTMENT = 0.015;
const PROJECT_AUTOSAVE_DELAY_MS = 1000;
const EXPORT_ERROR_TOAST_DURATION_MS = 20000;
@@ -677,6 +665,13 @@ export default function VideoEditor() {
const [selectedAnnotationId, setSelectedAnnotationId] = useState<string | null>(null);
const [audioRegions, setAudioRegions] = useState<AudioRegion[]>([]);
const [selectedAudioId, setSelectedAudioId] = useState<string | null>(null);
const [sourceAudioTrackSettingsByClip, setSourceAudioTrackSettingsByClip] = useState<
Record<string, SourceAudioTrackSettings>
>({});
const [defaultSourceAudioTrackSettings, setDefaultSourceAudioTrackSettings] = useState<
SourceAudioTrackSettings
>({});
const [hasClipSourceAudio, setHasClipSourceAudio] = useState(false);
const [autoCaptions, setAutoCaptions] = useState<CaptionCue[]>([]);
const [autoCaptionSettings, setAutoCaptionSettings] = useState<AutoCaptionSettings>(
DEFAULT_AUTO_CAPTION_SETTINGS,
@@ -700,9 +695,6 @@ export default function VideoEditor() {
const [exportError, setExportError] = useState<string | null>(null);
const [showExportDropdown, setShowExportDropdown] = useState(false);
const [previewVolume, setPreviewVolume] = useState(1);
const [sourceAudioFallbackPaths, setSourceAudioFallbackPaths] = useState<string[]>([]);
const [sourceAudioFallbackStartDelayMsByPath, setSourceAudioFallbackStartDelayMsByPath] =
useState<Record<string, number>>({});
const applySessionPresentation = useCallback(
(
session:
@@ -1767,6 +1759,8 @@ export default function VideoEditor() {
gifFrameRate: GifFrameRate;
gifLoop: boolean;
gifSizePreset: GifSizePreset;
sourceAudioTrackSettingsByClip: Record<string, SourceAudioTrackSettings>;
defaultSourceAudioTrackSettings: SourceAudioTrackSettings;
}>,
) => {
return editor;
@@ -1778,63 +1772,6 @@ export default function VideoEditor() {
() => videoSourcePath ?? (videoPath ? fromFileUrl(videoPath) : null),
[videoPath, videoSourcePath],
);
const { hasEmbeddedSourceAudio, externalAudioPaths: previewSourceAudioFallbackPaths } = useMemo(
() => resolveSourceAudioFallbackPaths(currentSourcePath, sourceAudioFallbackPaths),
[currentSourcePath, sourceAudioFallbackPaths],
);
const shouldMutePreviewVideo =
!hasEmbeddedSourceAudio && previewSourceAudioFallbackPaths.length > 0;
useEffect(() => {
let cancelled = false;
setSourceAudioFallbackPaths([]);
setSourceAudioFallbackStartDelayMsByPath({});
if (!currentSourcePath) {
return () => {
cancelled = true;
};
}
void (async () => {
try {
const result =
await window.electronAPI.getVideoAudioFallbackPaths(currentSourcePath);
if (cancelled) {
return;
}
if (!result.success) {
setSourceAudioFallbackPaths([]);
setSourceAudioFallbackStartDelayMsByPath({});
toast.warning(
result.error
? `Could not load companion audio sources: ${summarizeErrorMessage(result.error)}`
: "Could not load companion audio sources. Playback and export may miss microphone audio.",
{ id: SOURCE_AUDIO_FALLBACK_TOAST_ID, duration: 10000 },
);
return;
}
toast.dismiss(SOURCE_AUDIO_FALLBACK_TOAST_ID);
setSourceAudioFallbackPaths(result.paths ?? []);
setSourceAudioFallbackStartDelayMsByPath(result.startDelayMsByPath ?? {});
} catch (error) {
if (!cancelled) {
setSourceAudioFallbackPaths([]);
setSourceAudioFallbackStartDelayMsByPath({});
toast.warning(
`Could not load companion audio sources: ${summarizeErrorMessage(String(error))}`,
{ id: SOURCE_AUDIO_FALLBACK_TOAST_ID, duration: 10000 },
);
}
}
})();
return () => {
cancelled = true;
};
}, [currentSourcePath]);
const projectDisplayName = useMemo(() => {
const fileName =
currentProjectPath?.split(/[\\/]/).pop() ??
@@ -1925,6 +1862,8 @@ export default function VideoEditor() {
gifFrameRate,
gifLoop,
gifSizePreset,
sourceAudioTrackSettingsByClip,
defaultSourceAudioTrackSettings,
}),
[
buildPersistedEditorState,
@@ -1985,6 +1924,8 @@ export default function VideoEditor() {
gifLoop,
gifSizePreset,
frame,
sourceAudioTrackSettingsByClip,
defaultSourceAudioTrackSettings,
],
);
@@ -2170,6 +2111,10 @@ export default function VideoEditor() {
setSpeedRegions(normalizedEditor.speedRegions);
setAnnotationRegions(normalizedEditor.annotationRegions);
setAudioRegions(normalizedEditor.audioRegions);
setSourceAudioTrackSettingsByClip(normalizedEditor.sourceAudioTrackSettingsByClip ?? {});
setDefaultSourceAudioTrackSettings(
normalizedEditor.defaultSourceAudioTrackSettings ?? {},
);
setAutoCaptions(normalizedEditor.autoCaptions);
setAutoCaptionSettings(normalizedEditor.autoCaptionSettings);
setAspectRatio(normalizedEditor.aspectRatio);
@@ -3380,6 +3325,29 @@ export default function VideoEditor() {
}
return result;
}, [clipRegions, speedRegions]);
const audio = useVideoEditorAudio({
currentSourcePath,
selectedClipId,
clipRegions,
audioRegions,
effectiveSpeedRegions,
sourceAudioTrackSettingsByClip,
setSourceAudioTrackSettingsByClip,
defaultSourceAudioTrackSettings,
setDefaultSourceAudioTrackSettings,
currentTime,
timelineTime: timelinePlayheadTime,
duration,
isPlaying,
previewVolume,
summarizeErrorMessage,
onSourceFallbackLoadError: (error) => {
toast.warning(
`Could not load companion audio source: ${summarizeErrorMessage(getErrorMessage(error))}`,
{ duration: 10000 },
);
},
});
function togglePlayPause() {
const playback = videoPlaybackRef.current;
@@ -3761,6 +3729,18 @@ export default function VideoEditor() {
[selectedClipId],
);
const handleClipShowSourceAudioChange = useCallback(
(showSourceAudio: boolean) => {
if (!selectedClipId) return;
setClipRegions((prev) =>
prev.map((clip) =>
clip.id === selectedClipId ? { ...clip, showSourceAudio } : clip,
),
);
},
[selectedClipId],
);
const handleClipDelete = useCallback(
(id: string) => {
const deletedClip = clipRegions.find((clip) => clip.id === id);
@@ -3792,23 +3772,26 @@ export default function VideoEditor() {
if (id) {
setSelectedZoomId(null);
setSelectedAnnotationId(null);
setActiveEffectSection("audio");
}
}, []);
const handleAudioAdded = useCallback((span: Span, audioPath: string, trackIndex?: number) => {
const id = `audio-${nextAudioIdRef.current++}`;
const newRegion: AudioRegion = {
id,
startMs: Math.round(span.start),
endMs: Math.round(span.end),
audioPath,
volume: 1,
trackIndex,
};
const handleAudioAdded = useCallback((span: Span, audioPath: string, trackIndex?: number) => {
const id = `audio-${nextAudioIdRef.current++}`;
const newRegion: AudioRegion = {
id,
startMs: Math.round(span.start),
endMs: Math.round(span.end),
audioPath,
volume: 1,
normalize: false,
trackIndex,
};
setAudioRegions((prev) => [...prev, newRegion]);
setSelectedAudioId(id);
setSelectedZoomId(null);
setSelectedAnnotationId(null);
setActiveEffectSection("audio");
}, []);
const handleAudioSpanChange = useCallback((id: string, span: Span, trackIndex?: number) => {
@@ -3853,15 +3836,29 @@ export default function VideoEditor() {
[selectedAudioId],
);
const handleAudioDelete = useCallback(
(id: string) => {
const handleAudioDelete = useCallback(
(id: string) => {
setAudioRegions((prev) => prev.filter((region) => region.id !== id));
if (selectedAudioId === id) {
setSelectedAudioId(null);
}
},
[selectedAudioId],
);
[selectedAudioId],
);
const handleAudioNormalizeChange = useCallback(
(normalize: boolean) => {
if (!selectedAudioId) {
return;
}
setAudioRegions((prev) =>
prev.map((region) =>
region.id === selectedAudioId ? { ...region, normalize } : region,
),
);
},
[selectedAudioId],
);
const handleAnnotationAdded = useCallback((span: Span, trackIndex = 0) => {
const id = `annotation-${nextAnnotationIdRef.current++}`;
@@ -4096,290 +4093,6 @@ export default function VideoEditor() {
}
}, [selectedAudioId, audioRegions]);
// Audio playback sync: manage Audio elements that play in sync with video
const audioElementsRef = useRef<Map<string, HTMLAudioElement>>(new Map());
const audioElementRevokersRef = useRef<Map<string, () => void>>(new Map());
const audioElementResourcesRef = useRef<Map<string, string>>(new Map());
const sourceAudioElementsRef = useRef<Map<string, HTMLAudioElement>>(new Map());
const sourceAudioElementRevokersRef = useRef<Map<string, () => void>>(new Map());
const sourceAudioElementResourcesRef = useRef<Map<string, string>>(new Map());
const lastSourceAudioSyncTimeRef = useRef<number | null>(null);
useEffect(() => {
let cancelled = false;
const existing = audioElementsRef.current;
const currentIds = new Set(audioRegions.map((r) => r.id));
// Remove old audio elements
for (const [id, audio] of existing) {
if (!currentIds.has(id)) {
audio.pause();
audio.src = "";
audioElementRevokersRef.current.get(id)?.();
audioElementRevokersRef.current.delete(id);
audioElementResourcesRef.current.delete(id);
existing.delete(id);
}
}
// Create/update audio elements
for (const region of audioRegions) {
let audio = existing.get(region.id);
if (!audio) {
audio = new Audio();
audio.preload = "auto";
existing.set(region.id, audio);
}
if (audioElementResourcesRef.current.get(region.id) !== region.audioPath) {
audio.pause();
audio.src = "";
audioElementRevokersRef.current.get(region.id)?.();
audioElementRevokersRef.current.delete(region.id);
audioElementResourcesRef.current.set(region.id, region.audioPath);
void (async () => {
const resolved = await resolveMediaElementSource(region.audioPath);
const latestAudio = existing.get(region.id);
if (
cancelled ||
latestAudio !== audio ||
audioElementResourcesRef.current.get(region.id) !== region.audioPath
) {
resolved.revoke();
return;
}
audioElementRevokersRef.current.set(region.id, resolved.revoke);
latestAudio.src = resolved.src;
})();
}
audio.volume = Math.max(0, Math.min(1, region.volume * previewVolume));
}
return () => {
cancelled = true;
};
}, [audioRegions, previewVolume]);
useEffect(() => {
let cancelled = false;
const existing = sourceAudioElementsRef.current;
const currentIds = new Set(previewSourceAudioFallbackPaths);
for (const [id, audio] of existing) {
if (!currentIds.has(id)) {
audio.pause();
audio.src = "";
sourceAudioElementRevokersRef.current.get(id)?.();
sourceAudioElementRevokersRef.current.delete(id);
sourceAudioElementResourcesRef.current.delete(id);
existing.delete(id);
}
}
for (const audioPath of previewSourceAudioFallbackPaths) {
let audio = existing.get(audioPath);
if (!audio) {
audio = new Audio();
audio.preload = "auto";
existing.set(audioPath, audio);
}
audio.dataset.sourceAudioPath = audioPath;
if (sourceAudioElementResourcesRef.current.get(audioPath) !== audioPath) {
audio.pause();
audio.src = "";
sourceAudioElementRevokersRef.current.get(audioPath)?.();
sourceAudioElementRevokersRef.current.delete(audioPath);
sourceAudioElementResourcesRef.current.set(audioPath, audioPath);
void (async () => {
try {
const resolved = await resolveMediaElementSource(audioPath);
const latestAudio = existing.get(audioPath);
if (
cancelled ||
latestAudio !== audio ||
sourceAudioElementResourcesRef.current.get(audioPath) !== audioPath
) {
resolved.revoke();
return;
}
sourceAudioElementRevokersRef.current.set(audioPath, resolved.revoke);
latestAudio.src = resolved.src;
} catch (error) {
if (cancelled) {
return;
}
sourceAudioElementRevokersRef.current.get(audioPath)?.();
sourceAudioElementRevokersRef.current.delete(audioPath);
sourceAudioElementResourcesRef.current.delete(audioPath);
const latestAudio = existing.get(audioPath);
if (latestAudio === audio) {
latestAudio.pause();
latestAudio.src = "";
}
toast.warning(
`Could not load companion audio source: ${summarizeErrorMessage(getErrorMessage(error))}`,
{ id: SOURCE_AUDIO_FALLBACK_TOAST_ID, duration: 10000 },
);
}
})();
}
audio.volume = Math.max(0, Math.min(1, previewVolume));
}
if (previewSourceAudioFallbackPaths.length === 0) {
lastSourceAudioSyncTimeRef.current = null;
}
return () => {
cancelled = true;
};
}, [previewSourceAudioFallbackPaths, previewVolume]);
useEffect(() => {
return () => {
for (const audio of audioElementsRef.current.values()) {
audio.pause();
audio.src = "";
}
for (const revoke of audioElementRevokersRef.current.values()) {
revoke();
}
audioElementsRef.current.clear();
audioElementRevokersRef.current.clear();
audioElementResourcesRef.current.clear();
for (const audio of sourceAudioElementsRef.current.values()) {
audio.pause();
audio.src = "";
}
for (const revoke of sourceAudioElementRevokersRef.current.values()) {
revoke();
}
sourceAudioElementsRef.current.clear();
sourceAudioElementRevokersRef.current.clear();
sourceAudioElementResourcesRef.current.clear();
lastSourceAudioSyncTimeRef.current = null;
};
}, []);
// Sync audio playback with video currentTime and isPlaying state
useEffect(() => {
const currentTimeMs = currentTime * 1000;
const activeSpeedRegion = effectiveSpeedRegions.find(
(region) => currentTimeMs >= region.startMs && currentTimeMs < region.endMs,
);
const targetPlaybackRate = activeSpeedRegion ? activeSpeedRegion.speed : 1;
for (const region of audioRegions) {
const audio = audioElementsRef.current.get(region.id);
if (!audio) continue;
const isInRegion = currentTimeMs >= region.startMs && currentTimeMs < region.endMs;
if (isPlaying && isInRegion) {
enablePitchPreservingPlayback(audio);
const audioOffset = (currentTimeMs - region.startMs) / 1000;
// Only seek if significantly out of sync (> 200ms)
if (Math.abs(audio.currentTime - audioOffset) > 0.2) {
audio.currentTime = audioOffset;
}
const syncedPlaybackRate = getMediaSyncPlaybackRate({
basePlaybackRate: targetPlaybackRate,
currentTime: audio.currentTime,
targetTime: audioOffset,
});
if (Math.abs(audio.playbackRate - syncedPlaybackRate) > 0.001) {
audio.playbackRate = syncedPlaybackRate;
}
if (audio.paused) {
audio.play().catch(() => undefined);
}
} else {
if (!audio.paused) {
audio.pause();
}
}
}
}, [isPlaying, currentTime, audioRegions, effectiveSpeedRegions]);
useEffect(() => {
if (previewSourceAudioFallbackPaths.length === 0) {
lastSourceAudioSyncTimeRef.current = null;
return;
}
const activeSpeedRegion = effectiveSpeedRegions.find(
(region) => currentTime * 1000 >= region.startMs && currentTime * 1000 < region.endMs,
);
const targetPlaybackRate = activeSpeedRegion ? activeSpeedRegion.speed : 1;
const previousTimelineTime = lastSourceAudioSyncTimeRef.current;
const timelineJumped =
previousTimelineTime === null || Math.abs(currentTime - previousTimelineTime) > 0.25;
const driftThreshold = isPlaying
? SOURCE_AUDIO_PREVIEW_PLAYING_SEEK_DRIFT_SECONDS
: SOURCE_AUDIO_PREVIEW_PAUSED_SEEK_DRIFT_SECONDS;
for (const audio of sourceAudioElementsRef.current.values()) {
enablePitchPreservingPlayback(audio);
const audioDuration = Number.isFinite(audio.duration) ? audio.duration : null;
const startDelaySeconds = estimateCompanionAudioStartDelaySeconds(
duration,
audioDuration,
sourceAudioFallbackStartDelayMsByPath[audio.dataset.sourceAudioPath ?? ""],
);
const beforeAudioStart = currentTime + 0.001 < startDelaySeconds;
const targetTime = clampMediaTimeToDuration(
currentTime - startDelaySeconds,
audioDuration,
);
if (timelineJumped || Math.abs(audio.currentTime - targetTime) > driftThreshold) {
try {
audio.currentTime = targetTime;
} catch {
// no-op
}
}
const syncedPlaybackRate = getMediaSyncPlaybackRate({
basePlaybackRate: targetPlaybackRate,
currentTime: audio.currentTime,
targetTime,
toleranceSeconds: SOURCE_AUDIO_PREVIEW_RATE_TOLERANCE_SECONDS,
correctionWindowSeconds: SOURCE_AUDIO_PREVIEW_RATE_CORRECTION_WINDOW_SECONDS,
maxAdjustment: SOURCE_AUDIO_PREVIEW_MAX_RATE_ADJUSTMENT,
});
if (Math.abs(audio.playbackRate - syncedPlaybackRate) > 0.001) {
audio.playbackRate = syncedPlaybackRate;
}
const atEnd = audioDuration !== null && targetTime >= audioDuration;
if (isPlaying && !beforeAudioStart && !atEnd) {
audio.play().catch(() => undefined);
} else if (!audio.paused) {
audio.pause();
}
}
lastSourceAudioSyncTimeRef.current = currentTime;
}, [
currentTime,
duration,
isPlaying,
previewSourceAudioFallbackPaths,
sourceAudioFallbackStartDelayMsByPath,
effectiveSpeedRegions,
]);
const showExportSuccessToast = useCallback((filePath: string) => {
toast.success(`Exported successfully to ${filePath}`, {
action: {
@@ -4639,6 +4352,10 @@ export default function VideoEditor() {
encodingMode,
useModernNativeStaticLayout: useExperimentalNativeExport,
});
const sourceAudioTrackSettingsForExport =
selectedClipId !== null
? audio.selectedClipSourceAudioTrackSettings
: audio.activeSourceAudioTrackSettings;
const exporterConfig = {
videoUrl: videoPath,
@@ -4704,8 +4421,11 @@ export default function VideoEditor() {
cursorSway,
frame,
audioRegions,
sourceAudioFallbackPaths,
sourceAudioFallbackStartDelayMsByPath,
clipRegions,
sourceAudioFallbackPaths: audio.sourceAudioFallbackPaths,
sourceAudioFallbackStartDelayMsByPath:
audio.sourceAudioFallbackStartDelayMsByPath,
sourceAudioTrackSettings: sourceAudioTrackSettingsForExport,
previewWidth,
previewHeight,
onProgress: (progress: ExportProgress) => {
@@ -4952,8 +4672,10 @@ export default function VideoEditor() {
cursorClickBounceDuration,
cursorSway,
audioRegions,
sourceAudioFallbackPaths,
sourceAudioFallbackStartDelayMsByPath,
audio.sourceAudioFallbackPaths,
audio.sourceAudioFallbackStartDelayMsByPath,
audio.activeSourceAudioTrackSettings,
audio.selectedClipSourceAudioTrackSettings,
exportEncodingMode,
exportBackendPreference,
exportPipelineModel,
@@ -4985,6 +4707,7 @@ export default function VideoEditor() {
smokeExportConfig.shadowIntensity,
effectiveSpeedRegions,
frame,
selectedClipId,
smokeExportConfig.encodingMode,
smokeExportConfig.fps,
smokeExportConfig.quality,
@@ -6032,32 +5755,50 @@ export default function VideoEditor() {
selectedClipId={selectedClipId}
selectedClipSpeed={
selectedClipId
? (clipRegions.find((c) => c.id === selectedClipId)
?.speed ?? 1)
? clipRegions.find((c) => c.id === selectedClipId)?.speed ?? 1
: null
}
selectedClipMuted={
selectedClipId
? (clipRegions.find((c) => c.id === selectedClipId)
?.muted ?? false)
? clipRegions.find((c) => c.id === selectedClipId)?.muted ??
false
: null
}
onClipSpeedChange={(speed) =>
selectedClipId && handleClipSpeedChange(speed)
}
onClipMutedChange={(muted) =>
selectedClipId && handleClipMutedChange(muted)
selectedClipShowSourceAudio={
selectedClipId
? clipRegions.find((c) => c.id === selectedClipId)
?.showSourceAudio ?? false
: null
}
onClipSpeedChange={handleClipSpeedChange}
onClipMutedChange={handleClipMutedChange}
onClipShowSourceAudioChange={handleClipShowSourceAudioChange}
onClipDelete={handleClipDelete}
selectedAudioId={selectedAudioId}
selectedAudioVolume={
selectedAudioId
? (audioRegions.find((r) => r.id === selectedAudioId)
?.volume ?? null)
: null
hasClipSourceAudio={hasClipSourceAudio}
sourceAudioTrackMeta={audio.sourceAudioTrackMeta}
sourceAudioTrackSettings={audio.selectedClipSourceAudioTrackSettings}
onSourceAudioTrackVolumeChange={
audio.onSelectedClipSourceAudioTrackVolumeChange
}
onAudioVolumeChange={handleAudioVolumeChange}
onAudioDelete={handleAudioDelete}
onSourceAudioTrackNormalizeChange={
audio.onSelectedClipSourceAudioTrackNormalizeChange
}
selectedAudioId={selectedAudioId}
selectedAudioVolume={
selectedAudioId
? (audioRegions.find((r) => r.id === selectedAudioId)
?.volume ?? null)
: null
}
selectedAudioNormalize={
selectedAudioId
? (audioRegions.find((r) => r.id === selectedAudioId)
?.normalize ?? false)
: null
}
onAudioVolumeChange={handleAudioVolumeChange}
onAudioNormalizeChange={handleAudioNormalizeChange}
onAudioDelete={handleAudioDelete}
shadowIntensity={shadowIntensity}
onShadowChange={setShadowIntensity}
backgroundBlur={backgroundBlur}
@@ -6353,7 +6094,17 @@ export default function VideoEditor() {
cursorClickBounceDuration
}
cursorSway={cursorSway}
volume={shouldMutePreviewVideo ? 0 : previewVolume}
volume={
audio.shouldMutePreviewVideo || audio.isCurrentClipMuted
? 0
: Math.max(
0,
Math.min(
1,
previewVolume * audio.embeddedSourcePreviewGain,
),
)
}
suspendRendering={shouldSuspendPreviewRendering}
/>
</div>
@@ -6628,6 +6379,17 @@ export default function VideoEditor() {
selectedAnnotationId={selectedAnnotationId}
onSelectAnnotation={handleSelectAnnotation}
aspectRatio={aspectRatio}
showSourceAudioTrack={clipRegions.some((c) => c.showSourceAudio)}
sourceAudioTrackSettings={audio.activeSourceAudioTrackSettings}
getSourceAudioTrackSettingsForClip={
audio.getSourceAudioTrackSettingsForClip
}
onSourceAudioAvailabilityChange={(available) => {
setHasClipSourceAudio(available);
}}
onSourceAudioTracksMetaChange={(tracks) => {
audio.onSourceAudioTracksMetaChange(tracks);
}}
/>
</div>
</div>
@@ -0,0 +1,25 @@
import type { AudioPeaksData } from "../timeline/core/timelineTypes";
export type SourceAudioTrackId = "mixed" | "system" | "mic" | (string & {});
export interface SourceAudioTrackSetting {
volume: number;
normalize: boolean;
}
export type SourceAudioTrackSettings = Record<string, SourceAudioTrackSetting>;
export interface SourceAudioTrackMetaItem {
id: SourceAudioTrackId;
label: string;
}
export type SourceAudioTrackMeta = SourceAudioTrackMetaItem[];
export interface SourceAudioTrackWithPeaks extends SourceAudioTrackMetaItem {
peaks: AudioPeaksData;
}
export const SOURCE_AUDIO_FALLBACK_TOAST_ID = "source-audio-fallback-error";
export const SOURCE_AUDIO_NORMALIZE_GAIN = 1.35;
@@ -0,0 +1,18 @@
import { getClipSourceEndMs, sortClipRegions } from "../types";
import type { ClipRegion } from "../types";
export function getActiveClipIdAtSourceTime(
sourceTimeSeconds: number,
clipRegions: ClipRegion[],
): string | null {
const sourceMs = Math.round(sourceTimeSeconds * 1000);
const activeClip = sortClipRegions(clipRegions).find(
(clip) => sourceMs >= clip.startMs && sourceMs < getClipSourceEndMs(clip),
);
return activeClip?.id ?? null;
}
export function isClipMutedById(clipId: string | null, clipRegions: ClipRegion[]): boolean {
if (!clipId) return false;
return clipRegions.find((clip) => clip.id === clipId)?.muted ?? false;
}
@@ -0,0 +1,459 @@
import { useEffect, useMemo, useRef } from "react";
import { buildResolvedAudioPlan } from "@/lib/exporter/audioRoutingEngine";
import { resolveMediaElementSource } from "@/lib/exporter/localMediaSource";
import {
clampMediaTimeToDuration,
enablePitchPreservingPlayback,
estimateCompanionAudioStartDelaySeconds,
getMediaSyncPlaybackRate,
} from "@/lib/mediaTiming";
import type { AudioRegion, SpeedRegion } from "../types";
const SOURCE_AUDIO_PREVIEW_PLAYING_SEEK_DRIFT_SECONDS = 0.18;
const SOURCE_AUDIO_PREVIEW_PAUSED_SEEK_DRIFT_SECONDS = 0.01;
interface UseAudioPreviewSyncParams {
audioRegions: AudioRegion[];
previewVolume: number;
isPlaying: boolean;
currentTime: number;
timelineTime: number;
duration: number;
effectiveSpeedRegions: SpeedRegion[];
previewSourceAudioFallbackPaths: string[];
sourceAudioFallbackStartDelayMsByPath: Record<string, number>;
isCurrentClipMuted: boolean;
getSourceTrackPreviewGain: (audioPath: string) => number;
onSourceFallbackLoadError: (error: unknown) => void;
}
export function useAudioPreviewSync({
audioRegions,
previewVolume,
isPlaying,
currentTime,
timelineTime,
duration,
effectiveSpeedRegions,
previewSourceAudioFallbackPaths,
sourceAudioFallbackStartDelayMsByPath,
isCurrentClipMuted,
getSourceTrackPreviewGain,
onSourceFallbackLoadError,
}: UseAudioPreviewSyncParams) {
const resolvedPlan = useMemo(
() =>
buildResolvedAudioPlan({
videoResource: null,
sourceAudioFallbackPaths: previewSourceAudioFallbackPaths,
audioRegions,
}),
[audioRegions, previewSourceAudioFallbackPaths],
);
const resolvedUserTracks = useMemo(
() => resolvedPlan.tracks.filter((track) => track.kind === "user"),
[resolvedPlan],
);
const resolvedSourceTracks = useMemo(
() => resolvedPlan.tracks.filter((track) => track.kind !== "user"),
[resolvedPlan],
);
const audioElementsRef = useRef<Map<string, HTMLAudioElement>>(new Map());
const audioElementRevokersRef = useRef<Map<string, () => void>>(new Map());
const audioElementResourcesRef = useRef<Map<string, string>>(new Map());
const sourceAudioElementsRef = useRef<Map<string, HTMLAudioElement>>(new Map());
const sourceAudioMediaNodesRef = useRef<Map<string, MediaElementAudioSourceNode>>(new Map());
const sourceAudioGainNodesRef = useRef<Map<string, GainNode>>(new Map());
const sourceAudioElementRevokersRef = useRef<Map<string, () => void>>(new Map());
const sourceAudioElementResourcesRef = useRef<Map<string, string>>(new Map());
const sourceAudioContextRef = useRef<AudioContext | null>(null);
const sourceAudioMasterGainRef = useRef<GainNode | null>(null);
const sourceAudioResumePromiseRef = useRef<Promise<void> | null>(null);
const lastSourceAudioSyncTimeRef = useRef<number | null>(null);
const ensureSourceAudioContext = () => {
if (!sourceAudioContextRef.current) {
const context = new AudioContext({ latencyHint: "interactive" });
const masterGain = context.createGain();
masterGain.gain.value = 1;
masterGain.connect(context.destination);
sourceAudioContextRef.current = context;
sourceAudioMasterGainRef.current = masterGain;
}
return sourceAudioContextRef.current;
};
const ensureSourceAudioRunning = () => {
const context = ensureSourceAudioContext();
if (context.state === "running") {
return Promise.resolve();
}
if (!sourceAudioResumePromiseRef.current) {
sourceAudioResumePromiseRef.current = context
.resume()
.catch(() => undefined)
.finally(() => {
sourceAudioResumePromiseRef.current = null;
});
}
return sourceAudioResumePromiseRef.current;
};
useEffect(() => {
let cancelled = false;
const existing = audioElementsRef.current;
const currentIds = new Set(resolvedUserTracks.map((track) => track.id));
for (const [id, audio] of existing) {
if (!currentIds.has(id)) {
audio.pause();
audio.src = "";
audioElementRevokersRef.current.get(id)?.();
audioElementRevokersRef.current.delete(id);
audioElementResourcesRef.current.delete(id);
existing.delete(id);
}
}
for (const track of resolvedUserTracks) {
let audio = existing.get(track.id);
if (!audio) {
audio = new Audio();
audio.preload = "auto";
existing.set(track.id, audio);
}
if (audioElementResourcesRef.current.get(track.id) !== track.sourceRef.path) {
audio.pause();
audio.src = "";
audioElementRevokersRef.current.get(track.id)?.();
audioElementRevokersRef.current.delete(track.id);
audioElementResourcesRef.current.set(track.id, track.sourceRef.path);
void (async () => {
const resolved = await resolveMediaElementSource(track.sourceRef.path);
const latestAudio = existing.get(track.id);
if (
cancelled ||
latestAudio !== audio ||
audioElementResourcesRef.current.get(track.id) !== track.sourceRef.path
) {
resolved.revoke();
return;
}
audioElementRevokersRef.current.set(track.id, resolved.revoke);
latestAudio.src = resolved.src;
})();
}
audio.volume = Math.max(0, Math.min(1, track.gain * previewVolume));
}
return () => {
cancelled = true;
};
}, [previewVolume, resolvedUserTracks]);
useEffect(() => {
let cancelled = false;
const existing = sourceAudioElementsRef.current;
const currentIds = new Set(resolvedSourceTracks.map((track) => track.sourceRef.path));
for (const [id, audio] of existing) {
if (!currentIds.has(id)) {
audio.pause();
audio.src = "";
sourceAudioMediaNodesRef.current.get(id)?.disconnect();
sourceAudioMediaNodesRef.current.delete(id);
sourceAudioGainNodesRef.current.get(id)?.disconnect();
sourceAudioGainNodesRef.current.delete(id);
sourceAudioElementRevokersRef.current.get(id)?.();
sourceAudioElementRevokersRef.current.delete(id);
sourceAudioElementResourcesRef.current.delete(id);
existing.delete(id);
}
}
for (const track of resolvedSourceTracks) {
const audioPath = track.sourceRef.path;
let audio = existing.get(audioPath);
if (!audio) {
audio = new Audio();
audio.preload = "auto";
audio.crossOrigin = "anonymous";
existing.set(audioPath, audio);
}
audio.volume = 1;
audio.dataset.sourceAudioPath = audioPath;
const context = ensureSourceAudioContext();
const masterGain = sourceAudioMasterGainRef.current;
if (context && masterGain && !sourceAudioMediaNodesRef.current.has(audioPath)) {
try {
const mediaNode = context.createMediaElementSource(audio);
const trackGainNode = context.createGain();
mediaNode.connect(trackGainNode);
trackGainNode.connect(masterGain);
sourceAudioMediaNodesRef.current.set(audioPath, mediaNode);
sourceAudioGainNodesRef.current.set(audioPath, trackGainNode);
} catch (error) {
onSourceFallbackLoadError(error);
}
}
if (sourceAudioElementResourcesRef.current.get(audioPath) !== audioPath) {
audio.pause();
audio.src = "";
sourceAudioElementRevokersRef.current.get(audioPath)?.();
sourceAudioElementRevokersRef.current.delete(audioPath);
sourceAudioElementResourcesRef.current.set(audioPath, audioPath);
void (async () => {
try {
const resolved = await resolveMediaElementSource(audioPath);
const latestAudio = existing.get(audioPath);
if (
cancelled ||
latestAudio !== audio ||
sourceAudioElementResourcesRef.current.get(audioPath) !== audioPath
) {
resolved.revoke();
return;
}
sourceAudioElementRevokersRef.current.set(audioPath, resolved.revoke);
latestAudio.src = resolved.src;
} catch (error) {
if (cancelled) {
return;
}
sourceAudioElementRevokersRef.current.get(audioPath)?.();
sourceAudioElementRevokersRef.current.delete(audioPath);
sourceAudioElementResourcesRef.current.delete(audioPath);
const latestAudio = existing.get(audioPath);
if (latestAudio === audio) {
latestAudio.pause();
latestAudio.src = "";
}
onSourceFallbackLoadError(error);
}
})();
}
const trackGainNode = sourceAudioGainNodesRef.current.get(audioPath);
if (trackGainNode) {
trackGainNode.gain.value = Math.max(0, Math.min(2, getSourceTrackPreviewGain(audioPath)));
}
}
if (sourceAudioMasterGainRef.current) {
sourceAudioMasterGainRef.current.gain.value = isCurrentClipMuted
? 0
: Math.max(0, Math.min(1, previewVolume));
}
if (resolvedSourceTracks.length === 0) {
lastSourceAudioSyncTimeRef.current = null;
}
return () => {
cancelled = true;
};
}, [
getSourceTrackPreviewGain,
isCurrentClipMuted,
onSourceFallbackLoadError,
resolvedSourceTracks,
previewVolume,
]);
useEffect(() => {
return () => {
for (const audio of audioElementsRef.current.values()) {
audio.pause();
audio.src = "";
}
for (const revoke of audioElementRevokersRef.current.values()) {
revoke();
}
audioElementsRef.current.clear();
audioElementRevokersRef.current.clear();
audioElementResourcesRef.current.clear();
for (const audio of sourceAudioElementsRef.current.values()) {
audio.pause();
audio.src = "";
}
for (const node of sourceAudioMediaNodesRef.current.values()) {
node.disconnect();
}
for (const node of sourceAudioGainNodesRef.current.values()) {
node.disconnect();
}
for (const revoke of sourceAudioElementRevokersRef.current.values()) {
revoke();
}
sourceAudioElementsRef.current.clear();
sourceAudioMediaNodesRef.current.clear();
sourceAudioGainNodesRef.current.clear();
sourceAudioElementRevokersRef.current.clear();
sourceAudioElementResourcesRef.current.clear();
if (sourceAudioMasterGainRef.current) {
sourceAudioMasterGainRef.current.disconnect();
sourceAudioMasterGainRef.current = null;
}
const context = sourceAudioContextRef.current;
sourceAudioContextRef.current = null;
sourceAudioResumePromiseRef.current = null;
if (context) {
void context.close();
}
lastSourceAudioSyncTimeRef.current = null;
};
}, []);
useEffect(() => {
const currentTimeMs = timelineTime * 1000;
const activeSpeedRegion = effectiveSpeedRegions.find(
(region) => currentTimeMs >= region.startMs && currentTimeMs < region.endMs,
);
const targetPlaybackRate = activeSpeedRegion ? activeSpeedRegion.speed : 1;
for (const track of resolvedUserTracks) {
const audio = audioElementsRef.current.get(track.id);
if (!audio) continue;
const startMs = track.timelineBinding.startMs;
const endMs = track.timelineBinding.endMs;
const isInRegion = currentTimeMs >= startMs && currentTimeMs < endMs;
if (isPlaying && isInRegion) {
enablePitchPreservingPlayback(audio);
const audioOffset = (currentTimeMs - startMs) / 1000;
if (Math.abs(audio.currentTime - audioOffset) > 0.2) {
audio.currentTime = audioOffset;
}
const syncedPlaybackRate = getMediaSyncPlaybackRate({
basePlaybackRate: targetPlaybackRate,
currentTime: audio.currentTime,
targetTime: audioOffset,
});
if (Math.abs(audio.playbackRate - syncedPlaybackRate) > 0.001) {
audio.playbackRate = syncedPlaybackRate;
}
if (audio.paused) {
audio.play().catch(() => undefined);
}
} else if (!audio.paused) {
audio.pause();
}
}
}, [effectiveSpeedRegions, isPlaying, resolvedUserTracks, timelineTime]);
useEffect(() => {
if (resolvedSourceTracks.length === 0) {
lastSourceAudioSyncTimeRef.current = null;
return;
}
const activeSpeedRegion = effectiveSpeedRegions.find(
(region) => currentTime * 1000 >= region.startMs && currentTime * 1000 < region.endMs,
);
const targetPlaybackRate = activeSpeedRegion ? activeSpeedRegion.speed : 1;
const previousTimelineTime = lastSourceAudioSyncTimeRef.current;
const timelineJumped =
previousTimelineTime === null || Math.abs(currentTime - previousTimelineTime) > 0.25;
const driftThreshold = isPlaying
? SOURCE_AUDIO_PREVIEW_PLAYING_SEEK_DRIFT_SECONDS
: SOURCE_AUDIO_PREVIEW_PAUSED_SEEK_DRIFT_SECONDS;
if (sourceAudioMasterGainRef.current) {
sourceAudioMasterGainRef.current.gain.value = isCurrentClipMuted
? 0
: Math.max(0, Math.min(1, previewVolume));
}
for (const audio of sourceAudioElementsRef.current.values()) {
const sourceAudioPath = audio.dataset.sourceAudioPath ?? "";
const trackGainNode = sourceAudioGainNodesRef.current.get(sourceAudioPath);
if (trackGainNode) {
trackGainNode.gain.value = Math.max(0, Math.min(2, getSourceTrackPreviewGain(sourceAudioPath)));
}
enablePitchPreservingPlayback(audio);
const audioDuration = Number.isFinite(audio.duration) ? audio.duration : null;
const isMicCompanionTrack = /\.mic\./i.test(sourceAudioPath);
const rawStartDelaySeconds = estimateCompanionAudioStartDelaySeconds(
duration,
audioDuration,
sourceAudioFallbackStartDelayMsByPath[sourceAudioPath],
);
const maxPreviewStartDelaySeconds = isMicCompanionTrack ? 2 : 5;
const startDelaySeconds = isMicCompanionTrack
? 0
: Number.isFinite(duration) &&
(rawStartDelaySeconds >= Math.max(0, duration - 0.01) ||
rawStartDelaySeconds > Math.max(maxPreviewStartDelaySeconds, duration * 0.9))
? 0
: rawStartDelaySeconds;
const beforeAudioStart = currentTime + 0.001 < startDelaySeconds;
const targetTime = clampMediaTimeToDuration(currentTime - startDelaySeconds, audioDuration);
const shouldSeek =
timelineJumped ||
(!isPlaying && Math.abs(audio.currentTime - targetTime) > driftThreshold) ||
(isPlaying && Math.abs(audio.currentTime - targetTime) > 0.9);
if (shouldSeek) {
try {
audio.currentTime = targetTime;
} catch {
// no-op
}
}
// KISS for companion source tracks: fixed playback rate avoids audible flutter/stutter
// from continuous micro-corrections on system audio.
const syncedPlaybackRate = targetPlaybackRate;
if (Math.abs(audio.playbackRate - syncedPlaybackRate) > 0.001) {
audio.playbackRate = syncedPlaybackRate;
}
const atEnd = audioDuration !== null && targetTime >= audioDuration;
if (isPlaying && !beforeAudioStart && !atEnd) {
void ensureSourceAudioRunning().then(() => {
audio.play().catch(() => undefined);
});
} else if (!audio.paused) {
audio.pause();
}
}
lastSourceAudioSyncTimeRef.current = currentTime;
}, [
currentTime,
duration,
effectiveSpeedRegions,
getSourceTrackPreviewGain,
isCurrentClipMuted,
isPlaying,
previewVolume,
resolvedSourceTracks,
sourceAudioFallbackStartDelayMsByPath,
]);
useEffect(() => {
if (!isPlaying || resolvedSourceTracks.length === 0) {
return;
}
void ensureSourceAudioRunning().then(() => {
for (const audio of sourceAudioElementsRef.current.values()) {
if (audio.paused) {
audio.play().catch(() => undefined);
}
}
});
}, [isPlaying, resolvedSourceTracks.length]);
}
@@ -0,0 +1,93 @@
import React, { useCallback, useMemo } from "react";
import {
SOURCE_AUDIO_NORMALIZE_GAIN,
type SourceAudioTrackSettings,
} from "@/components/video-editor/audio/audioTypes";
import { useSourceAudioTrackSettings } from "./useSourceAudioTrackSettings";
import { getSourceTrackIdFromPath } from "@/lib/exporter/audioRoutingEngine";
interface UseClipAudioSettingsControllerParams {
selectedClipId: string | null;
activeClipId: string | null;
sourceAudioTrackSettingsByClip: Record<string, SourceAudioTrackSettings>;
setSourceAudioTrackSettingsByClip: React.Dispatch<
React.SetStateAction<Record<string, SourceAudioTrackSettings>>
>;
defaultSourceAudioTrackSettings: SourceAudioTrackSettings;
setDefaultSourceAudioTrackSettings: React.Dispatch<
React.SetStateAction<SourceAudioTrackSettings>
>;
}
export function useClipAudioSettingsController({
selectedClipId,
activeClipId,
sourceAudioTrackSettingsByClip,
setSourceAudioTrackSettingsByClip,
defaultSourceAudioTrackSettings,
setDefaultSourceAudioTrackSettings,
}: UseClipAudioSettingsControllerParams) {
const {
sourceAudioTrackMeta,
activeSourceAudioTrackSettings,
selectedClipSourceAudioTrackSettings,
getSourceAudioTrackSettingsForClip,
onSourceAudioTracksMetaChange,
onSelectedClipSourceAudioTrackVolumeChange,
onSelectedClipSourceAudioTrackNormalizeChange,
} = useSourceAudioTrackSettings({
selectedClipId,
activeClipId,
sourceAudioTrackSettingsByClip,
setSourceAudioTrackSettingsByClip,
defaultSourceAudioTrackSettings,
setDefaultSourceAudioTrackSettings,
});
const previewSourceAudioTrackSettings = useMemo(
() =>
activeClipId ? activeSourceAudioTrackSettings : selectedClipSourceAudioTrackSettings,
[activeClipId, activeSourceAudioTrackSettings, selectedClipSourceAudioTrackSettings],
);
const embeddedTrackId = useMemo<"mixed" | "system">(() => {
const hasMixedTrack = sourceAudioTrackMeta.some((track) => track.id === "mixed");
if (hasMixedTrack) return "mixed";
const hasSystemTrack = sourceAudioTrackMeta.some((track) => track.id === "system");
return hasSystemTrack ? "system" : "mixed";
}, [sourceAudioTrackMeta]);
const embeddedSourcePreviewGain = useMemo(() => {
const settings = previewSourceAudioTrackSettings[embeddedTrackId] ?? {
volume: 1,
normalize: false,
};
const normalizeGain = settings.normalize ? SOURCE_AUDIO_NORMALIZE_GAIN : 1;
return Math.max(0, Math.min(2, settings.volume * normalizeGain));
}, [embeddedTrackId, previewSourceAudioTrackSettings]);
const getSourceTrackPreviewGain = useCallback(
(audioPath: string) => {
const trackId = getSourceTrackIdFromPath(audioPath);
const settings = previewSourceAudioTrackSettings[trackId] ?? {
volume: 1,
normalize: false,
};
const normalizeGain = settings.normalize ? SOURCE_AUDIO_NORMALIZE_GAIN : 1;
return Math.max(0, Math.min(2, settings.volume * normalizeGain));
},
[previewSourceAudioTrackSettings],
);
return {
sourceAudioTrackMeta,
activeSourceAudioTrackSettings,
selectedClipSourceAudioTrackSettings,
getSourceAudioTrackSettingsForClip,
onSourceAudioTracksMetaChange,
onSelectedClipSourceAudioTrackVolumeChange,
onSelectedClipSourceAudioTrackNormalizeChange,
embeddedSourcePreviewGain,
getSourceTrackPreviewGain,
};
}
@@ -0,0 +1,68 @@
import { useEffect, useState } from "react";
import { toast } from "sonner";
import { SOURCE_AUDIO_FALLBACK_TOAST_ID } from "@/components/video-editor/audio/audioTypes";
interface UseSourceAudioFallbackParams {
currentSourcePath: string | null;
summarizeErrorMessage: (message: string) => string;
}
export function useSourceAudioFallback({
currentSourcePath,
summarizeErrorMessage,
}: UseSourceAudioFallbackParams) {
const [sourceAudioFallbackPaths, setSourceAudioFallbackPaths] = useState<string[]>([]);
const [sourceAudioFallbackStartDelayMsByPath, setSourceAudioFallbackStartDelayMsByPath] =
useState<Record<string, number>>({});
useEffect(() => {
let cancelled = false;
setSourceAudioFallbackPaths([]);
setSourceAudioFallbackStartDelayMsByPath({});
if (!currentSourcePath) {
return () => {
cancelled = true;
};
}
void (async () => {
try {
const result = await window.electronAPI.getVideoAudioFallbackPaths(currentSourcePath);
if (cancelled) {
return;
}
if (!result.success) {
setSourceAudioFallbackPaths([]);
setSourceAudioFallbackStartDelayMsByPath({});
toast.warning(
result.error
? `Could not load companion audio sources: ${summarizeErrorMessage(result.error)}`
: "Could not load companion audio sources. Playback and export may miss microphone audio.",
{ id: SOURCE_AUDIO_FALLBACK_TOAST_ID, duration: 10000 },
);
return;
}
toast.dismiss(SOURCE_AUDIO_FALLBACK_TOAST_ID);
setSourceAudioFallbackPaths(result.paths ?? []);
setSourceAudioFallbackStartDelayMsByPath(result.startDelayMsByPath ?? {});
} catch (error) {
if (!cancelled) {
setSourceAudioFallbackPaths([]);
setSourceAudioFallbackStartDelayMsByPath({});
toast.warning(
`Could not load companion audio sources: ${summarizeErrorMessage(String(error))}`,
{ id: SOURCE_AUDIO_FALLBACK_TOAST_ID, duration: 10000 },
);
}
}
})();
return () => {
cancelled = true;
};
}, [currentSourcePath, summarizeErrorMessage]);
return { sourceAudioFallbackPaths, sourceAudioFallbackStartDelayMsByPath };
}
@@ -0,0 +1,176 @@
import React, { useCallback, useMemo, useState } from "react";
import type {
SourceAudioTrackMeta,
SourceAudioTrackSettings,
} from "@/components/video-editor/audio/audioTypes";
interface UseSourceAudioTrackSettingsParams {
selectedClipId: string | null;
activeClipId: string | null;
sourceAudioTrackSettingsByClip: Record<string, SourceAudioTrackSettings>;
setSourceAudioTrackSettingsByClip: React.Dispatch<
React.SetStateAction<Record<string, SourceAudioTrackSettings>>
>;
defaultSourceAudioTrackSettings: SourceAudioTrackSettings;
setDefaultSourceAudioTrackSettings: React.Dispatch<React.SetStateAction<SourceAudioTrackSettings>>;
}
export interface UseSourceAudioTrackSettingsResult {
sourceAudioTrackMeta: SourceAudioTrackMeta;
activeSourceAudioTrackSettings: SourceAudioTrackSettings;
selectedClipSourceAudioTrackSettings: SourceAudioTrackSettings;
getSourceAudioTrackSettingsForClip: (clipId: string | null) => SourceAudioTrackSettings;
onSourceAudioTracksMetaChange: (tracks: SourceAudioTrackMeta) => void;
onSelectedClipSourceAudioTrackVolumeChange: (id: string, volume: number) => void;
onSelectedClipSourceAudioTrackNormalizeChange: (id: string, normalize: boolean) => void;
}
function isSameTrackMeta(left: SourceAudioTrackMeta, right: SourceAudioTrackMeta): boolean {
if (left.length !== right.length) return false;
for (let index = 0; index < left.length; index += 1) {
const leftTrack = left[index];
const rightTrack = right[index];
if (!leftTrack || !rightTrack) return false;
if (leftTrack.id !== rightTrack.id || leftTrack.label !== rightTrack.label) {
return false;
}
}
return true;
}
export function useSourceAudioTrackSettings({
selectedClipId,
activeClipId,
sourceAudioTrackSettingsByClip,
setSourceAudioTrackSettingsByClip,
defaultSourceAudioTrackSettings,
setDefaultSourceAudioTrackSettings,
}: UseSourceAudioTrackSettingsParams): UseSourceAudioTrackSettingsResult {
const [sourceAudioTrackMeta, setSourceAudioTrackMeta] = useState<SourceAudioTrackMeta>([]);
const activeSourceAudioTrackSettings = useMemo(() => {
if (!activeClipId) {
return defaultSourceAudioTrackSettings;
}
return {
...defaultSourceAudioTrackSettings,
...(sourceAudioTrackSettingsByClip[activeClipId] ?? {}),
};
}, [activeClipId, defaultSourceAudioTrackSettings, sourceAudioTrackSettingsByClip]);
const selectedClipSourceAudioTrackSettings = useMemo(() => {
if (!selectedClipId) {
return defaultSourceAudioTrackSettings;
}
return {
...defaultSourceAudioTrackSettings,
...(sourceAudioTrackSettingsByClip[selectedClipId] ?? {}),
};
}, [defaultSourceAudioTrackSettings, selectedClipId, sourceAudioTrackSettingsByClip]);
const onSourceAudioTracksMetaChange = useCallback((tracks: SourceAudioTrackMeta) => {
setSourceAudioTrackMeta((prev) => (isSameTrackMeta(prev, tracks) ? prev : tracks));
setDefaultSourceAudioTrackSettings((prev) => {
const next: SourceAudioTrackSettings = {};
for (const track of tracks) {
next[track.id] = prev[track.id] ?? { volume: 1, normalize: false };
}
const prevKeys = Object.keys(prev);
const nextKeys = Object.keys(next);
if (prevKeys.length !== nextKeys.length) {
return next;
}
for (const key of nextKeys) {
const prevSetting = prev[key];
const nextSetting = next[key];
if (!prevSetting || !nextSetting) {
return next;
}
if (
prevSetting.volume !== nextSetting.volume ||
prevSetting.normalize !== nextSetting.normalize
) {
return next;
}
}
return prev;
});
}, []);
const getSourceAudioTrackSettingsForClip = useCallback(
(clipId: string | null): SourceAudioTrackSettings => {
if (!clipId) {
return defaultSourceAudioTrackSettings;
}
return {
...defaultSourceAudioTrackSettings,
...(sourceAudioTrackSettingsByClip[clipId] ?? {}),
};
},
[defaultSourceAudioTrackSettings, sourceAudioTrackSettingsByClip],
);
const onSelectedClipSourceAudioTrackVolumeChange = useCallback(
(id: string, volume: number) => {
if (!selectedClipId) return;
setSourceAudioTrackSettingsByClip((prev) => {
const prevClip = prev[selectedClipId] ?? defaultSourceAudioTrackSettings;
const nextVolume = Number.isFinite(volume)
? Math.max(0, Math.min(2, volume))
: (prevClip[id]?.volume ?? 1);
const prevNormalize = prevClip[id]?.normalize ?? false;
if (
prevClip[id]?.volume === nextVolume &&
prevClip[id]?.normalize === prevNormalize
) {
return prev;
}
return {
...prev,
[selectedClipId]: {
...prevClip,
[id]: {
volume: nextVolume,
normalize: prevNormalize,
},
},
};
});
},
[defaultSourceAudioTrackSettings, selectedClipId],
);
const onSelectedClipSourceAudioTrackNormalizeChange = useCallback(
(id: string, normalize: boolean) => {
if (!selectedClipId) return;
setSourceAudioTrackSettingsByClip((prev) => {
const prevClip = prev[selectedClipId] ?? defaultSourceAudioTrackSettings;
const prevVolume = prevClip[id]?.volume ?? 1;
if (prevClip[id]?.normalize === normalize) {
return prev;
}
return {
...prev,
[selectedClipId]: {
...prevClip,
[id]: {
volume: prevVolume,
normalize,
},
},
};
});
},
[defaultSourceAudioTrackSettings, selectedClipId],
);
return {
sourceAudioTrackMeta,
activeSourceAudioTrackSettings,
selectedClipSourceAudioTrackSettings,
getSourceAudioTrackSettingsForClip,
onSourceAudioTracksMetaChange,
onSelectedClipSourceAudioTrackVolumeChange,
onSelectedClipSourceAudioTrackNormalizeChange,
};
}
@@ -0,0 +1,148 @@
import React, { useMemo } from "react";
import { resolveSourceTrackRoutingPolicy } from "@/lib/exporter/sourceTrackRoutingPolicy";
import type {
AudioRegion,
ClipRegion,
SpeedRegion,
} from "../types";
import type { SourceAudioTrackSettings } from "@/components/video-editor/audio/audioTypes";
import { getActiveClipIdAtSourceTime, isClipMutedById } from "./clipAudio";
import { useAudioPreviewSync } from "./useAudioPreviewSync";
import { useClipAudioSettingsController } from "./useClipAudioSettingsController";
import { useSourceAudioFallback } from "./useSourceAudioFallback";
function extractLocalPathFromMediaServerUrl(input: string | null | undefined): string | null {
if (!input) return null;
try {
const url = new URL(input);
const isLocalMediaServer =
(url.protocol === "http:" || url.protocol === "https:") &&
(url.hostname === "127.0.0.1" || url.hostname === "localhost") &&
url.pathname === "/video";
if (!isLocalMediaServer) return null;
return url.searchParams.get("path");
} catch {
return null;
}
}
interface UseVideoEditorAudioParams {
currentSourcePath: string | null;
selectedClipId: string | null;
clipRegions: ClipRegion[];
audioRegions: AudioRegion[];
effectiveSpeedRegions: SpeedRegion[];
sourceAudioTrackSettingsByClip: Record<string, SourceAudioTrackSettings>;
setSourceAudioTrackSettingsByClip: React.Dispatch<
React.SetStateAction<Record<string, SourceAudioTrackSettings>>
>;
defaultSourceAudioTrackSettings: SourceAudioTrackSettings;
setDefaultSourceAudioTrackSettings: React.Dispatch<
React.SetStateAction<SourceAudioTrackSettings>
>;
currentTime: number;
timelineTime: number;
duration: number;
isPlaying: boolean;
previewVolume: number;
summarizeErrorMessage: (message: string) => string;
onSourceFallbackLoadError: (error: unknown) => void;
}
export function useVideoEditorAudio({
currentSourcePath,
selectedClipId,
clipRegions,
audioRegions,
effectiveSpeedRegions,
sourceAudioTrackSettingsByClip,
setSourceAudioTrackSettingsByClip,
defaultSourceAudioTrackSettings,
setDefaultSourceAudioTrackSettings,
currentTime,
timelineTime,
duration,
isPlaying,
previewVolume,
summarizeErrorMessage,
onSourceFallbackLoadError,
}: UseVideoEditorAudioParams) {
const fallbackLookupSourcePath = useMemo(
() => extractLocalPathFromMediaServerUrl(currentSourcePath) ?? currentSourcePath,
[currentSourcePath],
);
const { sourceAudioFallbackPaths, sourceAudioFallbackStartDelayMsByPath } =
useSourceAudioFallback({
currentSourcePath: fallbackLookupSourcePath,
summarizeErrorMessage,
});
const sourceTrackRoutingPolicy = useMemo(
() => resolveSourceTrackRoutingPolicy(currentSourcePath, sourceAudioFallbackPaths),
[currentSourcePath, sourceAudioFallbackPaths],
);
const previewSourceAudioFallbackPaths = sourceTrackRoutingPolicy.playbackPaths;
const shouldMutePreviewVideo = sourceTrackRoutingPolicy.muteEmbeddedPreview;
const activeClipIdAtCurrentTime = useMemo(
() => getActiveClipIdAtSourceTime(currentTime, clipRegions),
[clipRegions, currentTime],
);
const isCurrentClipMuted = useMemo(
() => isClipMutedById(activeClipIdAtCurrentTime, clipRegions),
[activeClipIdAtCurrentTime, clipRegions],
);
const {
sourceAudioTrackMeta,
activeSourceAudioTrackSettings,
selectedClipSourceAudioTrackSettings,
getSourceAudioTrackSettingsForClip,
onSourceAudioTracksMetaChange,
onSelectedClipSourceAudioTrackVolumeChange,
onSelectedClipSourceAudioTrackNormalizeChange,
embeddedSourcePreviewGain,
getSourceTrackPreviewGain,
} = useClipAudioSettingsController({
selectedClipId,
activeClipId: activeClipIdAtCurrentTime,
sourceAudioTrackSettingsByClip,
setSourceAudioTrackSettingsByClip,
defaultSourceAudioTrackSettings,
setDefaultSourceAudioTrackSettings,
});
useAudioPreviewSync({
audioRegions,
previewVolume,
isPlaying,
currentTime,
timelineTime,
duration,
effectiveSpeedRegions,
previewSourceAudioFallbackPaths,
sourceAudioFallbackStartDelayMsByPath,
isCurrentClipMuted,
getSourceTrackPreviewGain,
onSourceFallbackLoadError,
});
return {
sourceAudioFallbackPaths,
sourceAudioFallbackStartDelayMsByPath,
previewSourceAudioFallbackPaths,
shouldMutePreviewVideo,
activeClipIdAtCurrentTime,
isCurrentClipMuted,
sourceAudioTrackMeta,
activeSourceAudioTrackSettings,
selectedClipSourceAudioTrackSettings,
getSourceAudioTrackSettingsForClip,
onSourceAudioTracksMetaChange,
onSelectedClipSourceAudioTrackVolumeChange,
onSelectedClipSourceAudioTrackNormalizeChange,
embeddedSourcePreviewGain,
getSourceTrackPreviewGain,
};
}
@@ -0,0 +1,113 @@
import WorkerConstructor from "./waveform.worker?worker";
import type { AudioPeaksData } from "../../timeline/core/timelineTypes";
import { WAVEFORM_DEFAULT_PEAK_COUNT } from "../../timeline/core/constants";
export class WaveformGenerator {
private audioContext: AudioContext;
private worker: Worker;
private peaksCache = new Map<string, AudioPeaksData>();
private pending = new Map<string, Promise<AudioPeaksData>>();
private workerRequestSeq = 0;
private workerResolvers = new Map<number, { resolve: (peaks: Float32Array) => void; reject: (err: Error) => void }>();
constructor() {
this.audioContext = new (window.AudioContext || (window as typeof window & { webkitAudioContext?: typeof AudioContext }).webkitAudioContext)();
this.worker = new WorkerConstructor();
this.worker.addEventListener(
"message",
(event: MessageEvent<{ requestId: number; peaks?: Float32Array; error?: string }>) => {
const { requestId, peaks, error } = event.data;
const resolver = this.workerResolvers.get(requestId);
if (!resolver) return;
this.workerResolvers.delete(requestId);
if (error) {
resolver.reject(new Error(error));
} else if (peaks) {
resolver.resolve(peaks);
}
},
);
this.worker.addEventListener("error", (error: ErrorEvent) => {
console.error("[WaveformGenerator] Worker fatal error:", error);
const fatalError = error.error ?? new Error(error.message || "Worker crashed");
// Reject all pending requests if the worker itself crashes
for (const resolver of this.workerResolvers.values()) {
resolver.reject(fatalError);
}
this.workerResolvers.clear();
});
}
private computePeaksWithWorker(channels: Float32Array[], samples: number): Promise<Float32Array> {
return new Promise((resolve, reject) => {
const requestId = ++this.workerRequestSeq;
this.workerResolvers.set(requestId, { resolve, reject });
this.worker.postMessage(
{
requestId,
channels,
samples,
},
channels.map(c => c.buffer),
);
});
}
public async generate(url: string, peakCount = WAVEFORM_DEFAULT_PEAK_COUNT): Promise<AudioPeaksData> {
const cacheKey = `${url}::${peakCount}`;
const cached = this.peaksCache.get(cacheKey);
if (cached) return cached;
const inflight = this.pending.get(cacheKey);
if (inflight) return inflight;
const request = (async () => {
const response = await fetch(url);
if (!response.ok) {
throw new Error(`Failed to load media: ${response.status}`);
}
const arrayBuffer = await response.arrayBuffer();
const decoded = await this.audioContext.decodeAudioData(arrayBuffer);
const channels: Float32Array[] = [];
for (let i = 0; i < decoded.numberOfChannels; i++) {
// We slice to transfer the underlying buffer to the worker
channels.push(decoded.getChannelData(i).slice());
}
const peaks = await this.computePeaksWithWorker(channels, peakCount);
let max = 0;
for (let i = 0; i < peaks.length; i++) {
if (peaks[i] > max) max = peaks[i];
}
if (max > 0) {
for (let i = 0; i < peaks.length; i++) {
peaks[i] /= max;
}
}
const result: AudioPeaksData = {
peaks,
durationMs: decoded.duration * 1000,
};
this.peaksCache.set(cacheKey, result);
this.pending.delete(cacheKey);
return result;
})().catch((error) => {
this.pending.delete(cacheKey);
throw error;
});
this.pending.set(cacheKey, request);
return request;
}
}
export const waveformGenerator = new WaveformGenerator();
@@ -0,0 +1,48 @@
type WaveformWorkerRequest = {
requestId: number;
channels: Float32Array[];
samples: number;
};
interface WorkerContext {
onmessage: (e: MessageEvent<WaveformWorkerRequest>) => void;
postMessage: (message: any, transfer?: Transferable[]) => void;
}
const workerScope = self as unknown as WorkerContext;
workerScope.onmessage = (e: MessageEvent<WaveformWorkerRequest>) => {
const { requestId, channels, samples } = e.data;
if (!channels || channels.length === 0 || samples <= 0) {
const empty = new Float32Array(0);
workerScope.postMessage({ requestId, peaks: empty }, [empty.buffer]);
return;
}
try {
const firstChannel = channels[0];
const result = new Float32Array(samples);
const total = firstChannel.length;
for (let i = 0; i < samples; i++) {
const start = Math.floor((i * total) / samples);
const end = Math.floor(((i + 1) * total) / samples);
let max = 0;
for (let j = start; j < end; j++) {
for (let c = 0; c < channels.length; c++) {
const val = Math.abs(channels[c][j]);
if (val > max) max = val;
}
}
result[i] = max;
}
workerScope.postMessage({ requestId, peaks: result }, [result.buffer]);
} catch (err) {
workerScope.postMessage({
requestId,
error: err instanceof Error ? err.message : "Unknown worker error",
});
}
};
@@ -20,6 +20,7 @@ import {
import { DEFAULT_WALLPAPER_PATH } from "@/lib/wallpapers";
import { ASPECT_RATIOS, type AspectRatio, isCustomAspectRatio } from "@/utils/aspectRatioUtils";
import { CURSOR_MOTION_PRESETS, resolveCursorMotionPresetId } from "./cursorMotionPresets";
import type { SourceAudioTrackSettings } from "@/components/video-editor/audio/audioTypes";
import {
type AnnotationRegion,
type AudioRegion,
@@ -127,6 +128,8 @@ export interface ProjectEditorState {
autoCaptionSettings: AutoCaptionSettings;
webcam: WebcamOverlaySettings;
aspectRatio: AspectRatio;
sourceAudioTrackSettingsByClip?: Record<string, SourceAudioTrackSettings>;
defaultSourceAudioTrackSettings?: SourceAudioTrackSettings;
exportEncodingMode: ExportEncodingMode;
exportBackendPreference: ExportBackendPreference;
exportPipelineModel: ExportPipelineModel;
@@ -496,6 +499,10 @@ export function normalizeProjectEditor(editor: Partial<ProjectEditorState>): Pro
endMs,
speed: isFiniteNumber(region.speed) ? region.speed : 1,
muted: typeof region.muted === "boolean" ? region.muted : false,
showSourceAudio:
typeof region.showSourceAudio === "boolean"
? region.showSourceAudio
: false,
};
})
: [];
@@ -647,16 +654,17 @@ export function normalizeProjectEditor(editor: Partial<ProjectEditorState>): Pro
const startMs = Math.max(0, Math.min(rawStart, rawEnd));
const endMs = Math.max(startMs + 1, rawEnd);
return {
id: region.id,
startMs,
endMs,
audioPath: typeof region.audioPath === "string" ? region.audioPath : "",
volume: isFiniteNumber(region.volume) ? clamp(region.volume, 0, 1) : 1,
trackIndex: isFiniteNumber(region.trackIndex)
? Math.max(0, Math.floor(region.trackIndex))
: 0,
};
return {
id: region.id,
startMs,
endMs,
audioPath: typeof region.audioPath === "string" ? region.audioPath : "",
volume: isFiniteNumber(region.volume) ? clamp(region.volume, 0, 1) : 1,
normalize: Boolean(region.normalize),
trackIndex: isFiniteNumber(region.trackIndex)
? Math.max(0, Math.floor(region.trackIndex))
: 0,
};
})
: [];
@@ -983,6 +991,16 @@ export function normalizeProjectEditor(editor: Partial<ProjectEditorState>): Pro
? clamp(webcam.margin, 0, 96)
: DEFAULT_WEBCAM_MARGIN,
},
sourceAudioTrackSettingsByClip:
editor.sourceAudioTrackSettingsByClip &&
typeof editor.sourceAudioTrackSettingsByClip === "object"
? editor.sourceAudioTrackSettingsByClip
: {},
defaultSourceAudioTrackSettings:
editor.defaultSourceAudioTrackSettings &&
typeof editor.defaultSourceAudioTrackSettings === "object"
? editor.defaultSourceAudioTrackSettings
: {},
aspectRatio:
typeof editor.aspectRatio === "string" &&
(validAspectRatios.has(editor.aspectRatio as AspectRatio) ||
@@ -5,18 +5,22 @@ import {
MusicNotes as Music,
MouseLeftClickIcon as PhMouseLeftClick,
Scissors,
SpeakerX,
MagnifyingGlassPlus as ZoomIn,
} from "@phosphor-icons/react";
import type { Span } from "dnd-timeline";
import { useItem } from "dnd-timeline";
import { useMemo } from "react";
import { cn } from "@/lib/utils";
import AudioWaveform from "./components/waveform/AudioWaveform";
import type { AudioPeaksData } from "./core/timelineTypes";
import glassStyles from "./ItemGlass.module.css";
interface ItemProps {
id: string;
span: Span;
rowId: string;
disabled?: boolean;
children: React.ReactNode;
isSelected?: boolean;
onSelect?: () => void;
@@ -24,6 +28,11 @@ interface ItemProps {
zoomDepth?: number;
zoomMode?: "auto" | "manual";
speedValue?: number;
waveformPeaks?: AudioPeaksData | null;
waveformSegmentSpan?: Span;
waveformGain?: number;
waveformNormalize?: boolean;
muted?: boolean;
variant?: "zoom" | "trim" | "clip" | "annotation" | "speed" | "audio";
}
@@ -51,18 +60,25 @@ export default function Item({
id,
span,
rowId,
disabled = false,
isSelected = false,
onSelect,
onSelectId,
zoomDepth = 1,
zoomMode = "auto",
speedValue,
waveformPeaks = null,
waveformSegmentSpan,
waveformGain = 1,
waveformNormalize = false,
muted = false,
variant = "zoom",
children,
}: ItemProps) {
const { setNodeRef, attributes, listeners, itemStyle, itemContentStyle } = useItem({
id,
span,
disabled,
data: { rowId },
});
@@ -71,6 +87,7 @@ export default function Item({
const isClip = variant === "clip";
const isSpeed = variant === "speed";
const isAudio = variant === "audio";
const showAudioWaveform = isAudio && Boolean(waveformPeaks);
const glassClass = isZoom
? glassStyles.glassPurple
@@ -146,6 +163,22 @@ export default function Item({
style={{ cursor: "col-resize", pointerEvents: "auto" }}
title="Resize right"
/>
{showAudioWaveform && waveformPeaks && (
<AudioWaveform
peaks={waveformPeaks}
segmentStartMs={waveformSegmentSpan?.start ?? span.start}
segmentEndMs={waveformSegmentSpan?.end ?? span.end}
gain={waveformGain}
normalize={waveformNormalize}
className="absolute inset-0 w-full h-full pointer-events-none opacity-45"
/>
)}
{/* Muted overlay for source audio track items */}
{isAudio && muted && (
<div className="absolute inset-0 z-20 flex items-center justify-center gap-1 bg-red-900/40 pointer-events-none">
<SpeakerX className="w-3 h-3 text-red-300/90 shrink-0" />
</div>
)}
{/* Content */}
<div className="relative z-10 flex flex-col items-center justify-center text-black/70 dark:text-white/90 opacity-80 group-hover:opacity-100 transition-opacity select-none overflow-hidden">
<div className="flex items-center gap-1.5">
@@ -17,6 +17,12 @@ import {
} from "@/utils/aspectRatioUtils";
import { formatShortcut } from "@/utils/platformUtils";
import { loadEditorPreferences, saveEditorPreferences } from "../editorPreferences";
import { fromFileUrl } from "../projectPersistence";
import type {
SourceAudioTrackMeta,
SourceAudioTrackSettings,
SourceAudioTrackWithPeaks,
} from "@/components/video-editor/audio/audioTypes";
import type {
AnnotationRegion,
AudioRegion,
@@ -80,6 +86,38 @@ export interface TimelineEditorProps {
isCropped?: boolean;
videoPath?: string | null;
hideToolbar?: boolean;
showSourceAudioTrack?: boolean;
onSourceAudioAvailabilityChange?: (available: boolean) => void;
sourceAudioTrackSettings?: SourceAudioTrackSettings;
getSourceAudioTrackSettingsForClip?: (
clipId: string | null,
) => SourceAudioTrackSettings;
onSourceAudioTracksMetaChange?: (tracks: SourceAudioTrackMeta) => void;
}
function extractLocalPathFromMediaServerUrl(input: string | null | undefined): string | null {
if (!input) return null;
try {
const url = new URL(input);
const isLocalMediaServer =
(url.protocol === "http:" || url.protocol === "https:") &&
(url.hostname === "127.0.0.1" || url.hostname === "localhost") &&
url.pathname === "/video";
if (!isLocalMediaServer) return null;
return url.searchParams.get("path");
} catch {
return null;
}
}
function buildSourceSidecarPath(source: string, suffix: "mic" | "system"): string {
const normalized = source.replace(/\\/g, "/");
const lastSlash = normalized.lastIndexOf("/");
const dir = lastSlash >= 0 ? normalized.slice(0, lastSlash + 1) : "";
const fileName = lastSlash >= 0 ? normalized.slice(lastSlash + 1) : normalized;
const dotIndex = fileName.lastIndexOf(".");
const baseName = dotIndex > 0 ? fileName.slice(0, dotIndex) : fileName;
return `${dir}${baseName}.${suffix}.wav`;
}
export interface TimelineEditorHandle {
@@ -138,6 +176,11 @@ const TimelineEditor = forwardRef<TimelineEditorHandle, TimelineEditorProps>(
isCropped = false,
videoPath,
hideToolbar = false,
showSourceAudioTrack = false,
onSourceAudioAvailabilityChange,
sourceAudioTrackSettings = {},
getSourceAudioTrackSettingsForClip,
onSourceAudioTracksMetaChange,
},
ref,
) {
@@ -178,8 +221,111 @@ const TimelineEditor = forwardRef<TimelineEditorHandle, TimelineEditorProps>(
pan: "Shift + Ctrl + Scroll",
zoom: "Ctrl + Scroll",
});
const [liveSpanPreviewById, setLiveSpanPreviewById] = useState<Record<string, Span>>({});
const liveZoomPreview = useMemo(() => {
const previewSpans: Record<string, Span> = { ...liveSpanPreviewById };
const hiddenZoomIds = new Set<string>();
for (const [previewId, previewSpan] of Object.entries(liveSpanPreviewById)) {
const oldClip = clipRegions.find((clip) => clip.id === previewId);
if (!oldClip) continue;
const newStart = Math.round(previewSpan.start);
const newEnd = Math.round(previewSpan.end);
const removedSegments = [
...(newStart > oldClip.startMs
? [{ startMs: oldClip.startMs, endMs: newStart }]
: []),
...(newEnd < oldClip.endMs
? [{ startMs: newEnd, endMs: oldClip.endMs }]
: []),
];
const startDelta = newStart - oldClip.startMs;
const endDelta = newEnd - oldClip.endMs;
const isMove = Math.abs(startDelta - endDelta) < 1 && Math.abs(startDelta) > 0;
if (isMove) {
const delta = startDelta;
for (const zoom of zoomRegions) {
const overlaps =
zoom.startMs < oldClip.endMs && zoom.endMs > oldClip.startMs;
if (!overlaps) continue;
previewSpans[zoom.id] = {
start: zoom.startMs + delta,
end: zoom.endMs + delta,
};
}
}
if (removedSegments.length > 0) {
for (const zoom of zoomRegions) {
const removed = removedSegments.some(
(segment) =>
zoom.startMs < segment.endMs && zoom.endMs > segment.startMs,
);
if (removed) hiddenZoomIds.add(zoom.id);
}
}
}
return { previewSpans, hiddenZoomIds };
}, [clipRegions, liveSpanPreviewById, zoomRegions]);
const { shortcuts: keyShortcuts, isMac } = useShortcuts();
const audioPeaks = useTimelineAudioPeaks(videoPath);
const sourceAudioPeaks = useTimelineAudioPeaks(videoPath, {
enableSourceSidecarFallback: true,
});
const localSourcePath = useMemo(() => {
if (!videoPath) return null;
return (
extractLocalPathFromMediaServerUrl(videoPath) ||
(/^file:\/\//i.test(videoPath) ? fromFileUrl(videoPath) : videoPath)
);
}, [videoPath]);
const micSidecarPath = useMemo(
() => (localSourcePath ? buildSourceSidecarPath(localSourcePath, "mic") : null),
[localSourcePath],
);
const systemSidecarPath = useMemo(
() => (localSourcePath ? buildSourceSidecarPath(localSourcePath, "system") : null),
[localSourcePath],
);
const micSidecarPeaks = useTimelineAudioPeaks(micSidecarPath);
const systemSidecarPeaks = useTimelineAudioPeaks(systemSidecarPath);
const sourceAudioTracks = useMemo<SourceAudioTrackWithPeaks[]>(() => {
if (systemSidecarPeaks || micSidecarPeaks) {
const tracks: SourceAudioTrackWithPeaks[] = [];
if (systemSidecarPeaks)
tracks.push({
id: "system",
label: t("audio.systemLabel", "Source System"),
peaks: systemSidecarPeaks,
});
if (micSidecarPeaks)
tracks.push({
id: "mic",
label: t("audio.micLabel", "Source Mic"),
peaks: micSidecarPeaks,
});
return tracks;
}
return sourceAudioPeaks
? [
{
id: "mixed",
label: t("audio.mixedLabel", "Source"),
peaks: sourceAudioPeaks,
},
]
: [];
}, [micSidecarPeaks, sourceAudioPeaks, systemSidecarPeaks, t]);
useEffect(() => {
onSourceAudioTracksMetaChange?.(sourceAudioTracks.map((t) => ({ id: t.id, label: t.label })));
}, [onSourceAudioTracksMetaChange, sourceAudioTracks]);
void sourceAudioTrackSettings;
useEffect(() => {
onSourceAudioAvailabilityChange?.(sourceAudioTracks.length > 0);
}, [onSourceAudioAvailabilityChange, sourceAudioTracks.length]);
useEffect(() => {
if (aspectRatio === "native") {
@@ -376,6 +522,25 @@ const TimelineEditor = forwardRef<TimelineEditorHandle, TimelineEditorProps>(
onItemSpanChange={handleItemSpanChange}
resolveTargetRowId={getResolvedDropRowId}
allRegionSpans={allRegionSpans}
onLiveSpanPreviewChange={(id, span) => {
setLiveSpanPreviewById((prev) => {
if (!span) {
if (!(id in prev)) return prev;
const next = { ...prev };
delete next[id];
return next;
}
const current = prev[id];
if (
current &&
current.start === span.start &&
current.end === span.end
) {
return prev;
}
return { ...prev, [id]: span };
});
}}
>
<KeyframeMarkers
keyframes={keyframes}
@@ -403,7 +568,11 @@ const TimelineEditor = forwardRef<TimelineEditorHandle, TimelineEditorProps>(
selectAllBlocksActive={selectAllBlocksActive}
onClearBlockSelection={clearSelectedBlocks}
keyframes={keyframes}
audioPeaks={audioPeaks}
sourceAudioTracks={sourceAudioTracks}
getSourceAudioTrackSettingsForClip={getSourceAudioTrackSettingsForClip}
showSourceAudioTrack={showSourceAudioTrack}
liveSpanPreviewById={liveZoomPreview.previewSpans}
liveHiddenItemIds={Array.from(liveZoomPreview.hiddenZoomIds)}
/>
</TimelineWrapper>
</div>
@@ -11,18 +11,21 @@ import {
type MouseEventHandler,
} from "react";
import { cn } from "@/lib/utils";
import type {
SourceAudioTrackSettings,
SourceAudioTrackWithPeaks,
} from "@/components/video-editor/audio/audioTypes";
import {
getTimelineContentMinHeightPx,
getTimelineRowsMinHeightPx,
getTimelineViewportStretchFactor,
TIMELINE_AXIS_HEIGHT_PX,
} from "../../timelineLayout";
import AudioWaveform from "../waveform/AudioWaveform";
import glassStyles from "../../ItemGlass.module.css";
import Item from "../../Item";
import Row from "../../Row";
import { CLIP_ROW_ID, ZOOM_ROW_ID } from "../../core/constants";
import type { AudioPeaksData, TimelineRenderItem } from "../../core/timelineTypes";
import { CLIP_ROW_ID, SOURCE_AUDIO_ROW_ID, ZOOM_ROW_ID } from "../../core/constants";
import type { TimelineRenderItem } from "../../core/timelineTypes";
import {
getAnnotationTrackIndex,
getAnnotationTrackRowId,
@@ -34,6 +37,7 @@ import {
import TimelineAxis from "../axis/TimelineAxis";
import ClipMarkerOverlay from "../overlays/ClipMarkerOverlay";
import PlaybackCursor from "../playhead/PlaybackCursor";
import { useTimelineAudioPeaks } from "../../hooks/useTimelineAudioPeaks";
const HINT_CLIP = "Press C to split clip";
const HINT_ANNOTATION = "Press A to add annotation";
@@ -57,7 +61,13 @@ interface TimelineCanvasProps {
selectAllBlocksActive?: boolean;
onClearBlockSelection?: () => void;
keyframes?: { id: string; time: number }[];
audioPeaks?: AudioPeaksData | null;
sourceAudioTracks?: SourceAudioTrackWithPeaks[];
getSourceAudioTrackSettingsForClip?: (
clipId: string | null,
) => SourceAudioTrackSettings;
showSourceAudioTrack?: boolean;
liveSpanPreviewById?: Record<string, { start: number; end: number }>;
liveHiddenItemIds?: string[];
}
interface TimelineHoverParams {
@@ -218,7 +228,13 @@ interface TimelineCanvasRowsProps {
onSelectClip?: (id: string | null) => void;
onSelectAnnotation?: (id: string | null) => void;
onSelectAudio?: (id: string | null) => void;
audioPeaks?: AudioPeaksData | null;
sourceAudioTracks?: SourceAudioTrackWithPeaks[];
getSourceAudioTrackSettingsForClip?: (
clipId: string | null,
) => SourceAudioTrackSettings;
showSourceAudioTrack?: boolean;
liveSpanPreviewById?: Record<string, { start: number; end: number }>;
liveHiddenItemIds?: string[];
direction: string;
canShowGhostZoom: boolean;
ghostStartMs: number | null;
@@ -230,6 +246,44 @@ interface TimelineCanvasRowsProps {
onZoomRowClick: MouseEventHandler<HTMLDivElement>;
}
interface AudioItemWithWaveformProps {
item: TimelineRenderItem;
span: { start: number; end: number };
waveformSpan: { start: number; end: number };
isSelected: boolean;
onSelectAudio?: (id: string | null) => void;
}
function AudioItemWithWaveform({
item,
span,
waveformSpan,
isSelected,
onSelectAudio,
}: AudioItemWithWaveformProps) {
const peaks = useTimelineAudioPeaks(item.audioPath ?? null);
const normalizedWaveformSpan = useMemo(() => {
const duration = Math.max(0, waveformSpan.end - waveformSpan.start);
return { start: 0, end: duration };
}, [waveformSpan.end, waveformSpan.start]);
return (
<Item
id={item.id}
rowId={item.rowId}
span={span}
isSelected={isSelected}
onSelectId={onSelectAudio}
variant="audio"
waveformPeaks={peaks}
waveformSegmentSpan={normalizedWaveformSpan}
waveformGain={Math.max(0, Math.min(2, item.audioGain ?? 1))}
waveformNormalize={Boolean(item.audioNormalize)}
>
{item.label}
</Item>
);
}
const TimelineCanvasRows = memo(function TimelineCanvasRows({
items,
videoDurationMs,
@@ -242,7 +296,11 @@ const TimelineCanvasRows = memo(function TimelineCanvasRows({
onSelectClip,
onSelectAnnotation,
onSelectAudio,
audioPeaks,
sourceAudioTracks = [],
getSourceAudioTrackSettingsForClip,
showSourceAudioTrack = false,
liveSpanPreviewById,
liveHiddenItemIds,
direction,
canShowGhostZoom,
ghostStartMs,
@@ -253,6 +311,7 @@ const TimelineCanvasRows = memo(function TimelineCanvasRows({
onZoomRowMouseLeave,
onZoomRowClick,
}: TimelineCanvasRowsProps) {
const hiddenIds = useMemo(() => new Set(liveHiddenItemIds ?? []), [liveHiddenItemIds]);
const { clipItems, zoomItems, annotationRows, audioRows } = useMemo(() => {
const nextClipItems: TimelineRenderItem[] = [];
const nextZoomItems: TimelineRenderItem[] = [];
@@ -307,7 +366,6 @@ const TimelineCanvasRows = memo(function TimelineCanvasRows({
return (
<>
<Row id={CLIP_ROW_ID} isEmpty={clipItems.length === 0} hint={HINT_CLIP}>
{audioPeaks && <AudioWaveform peaks={audioPeaks} />}
<ClipMarkerOverlay videoDurationMs={videoDurationMs} />
{clipItems.map((item) => (
<Item
@@ -323,6 +381,35 @@ const TimelineCanvasRows = memo(function TimelineCanvasRows({
</Item>
))}
</Row>
{showSourceAudioTrack &&
sourceAudioTracks.map((track) => (
<Row key={track.id} id={`${SOURCE_AUDIO_ROW_ID}-${track.id}`}>
{clipItems.filter(item => item.showSourceAudio).map((item) => {
const settings = getSourceAudioTrackSettingsForClip?.(item.id)?.[
track.id
] ?? { volume: 1, normalize: false };
return (
<Item
key={`source-audio-${track.id}-${item.id}`}
id={`source-audio-${track.id}-${item.id}`}
rowId={`${SOURCE_AUDIO_ROW_ID}-${track.id}`}
span={liveSpanPreviewById?.[item.id] ?? item.span}
disabled
isSelected={selectAllBlocksActive || item.id === selectedClipId}
onSelect={() => onSelectClip?.(item.id)}
variant="audio"
waveformPeaks={track.peaks}
waveformSegmentSpan={item.sourceSpan ?? item.span}
waveformGain={Math.max(0, Math.min(2, settings.volume))}
waveformNormalize={Boolean(settings.normalize)}
muted={item.muted}
>
{track.label}
</Item>
);
})}
</Row>
))}
<Row
id={ZOOM_ROW_ID}
@@ -357,7 +444,9 @@ const TimelineCanvasRows = memo(function TimelineCanvasRows({
</div>
</div>
)}
{zoomItems.map((item) => (
{zoomItems
.filter((item) => !hiddenIds.has(item.id))
.map((item) => (
<Item
id={item.id}
key={item.id}
@@ -395,17 +484,14 @@ const TimelineCanvasRows = memo(function TimelineCanvasRows({
{audioRows.map(({ rowId, items: rowItems }, index) => (
<Row key={rowId} id={rowId} isEmpty={rowItems.length === 0} hint={index === 0 ? HINT_AUDIO : undefined}>
{rowItems.map((item) => (
<Item
id={item.id}
<AudioItemWithWaveform
key={item.id}
rowId={item.rowId}
item={item}
span={item.span}
waveformSpan={liveSpanPreviewById?.[item.id] ?? item.span}
isSelected={selectAllBlocksActive || item.id === selectedAudioId}
onSelectId={onSelectAudio}
variant="audio"
>
{item.label}
</Item>
onSelectAudio={onSelectAudio}
/>
))}
</Row>
))}
@@ -431,7 +517,11 @@ export default function TimelineCanvas({
selectAllBlocksActive = false,
onClearBlockSelection,
keyframes = [],
audioPeaks,
sourceAudioTracks = [],
getSourceAudioTrackSettingsForClip,
showSourceAudioTrack = false,
liveSpanPreviewById,
liveHiddenItemIds,
}: TimelineCanvasProps) {
const { setTimelineRef, style, sidebarWidth, direction, range, valueToPixels, pixelsToValue } =
useTimelineContext();
@@ -583,8 +673,9 @@ export default function TimelineCanvas({
if (isAnnotationTrackRowId(item.rowId)) annotationRowIds.add(item.rowId);
if (isAudioTrackRowId(item.rowId)) audioRowIds.add(item.rowId);
}
return 2 + annotationRowIds.size + audioRowIds.size;
}, [items]);
const sourceAudioRows = showSourceAudioTrack ? sourceAudioTracks.length : 0;
return 2 + sourceAudioRows + annotationRowIds.size + audioRowIds.size;
}, [items, showSourceAudioTrack, sourceAudioTracks.length]);
const timelineRowsMinHeightPx = getTimelineRowsMinHeightPx(timelineRowCount);
const timelineContentMinHeightPx = getTimelineContentMinHeightPx(timelineRowCount);
const timelineViewportStretchFactor = getTimelineViewportStretchFactor(timelineRowCount);
@@ -660,7 +751,11 @@ export default function TimelineCanvas({
onSelectClip={onSelectClip}
onSelectAnnotation={onSelectAnnotation}
onSelectAudio={onSelectAudio}
audioPeaks={audioPeaks}
sourceAudioTracks={sourceAudioTracks}
getSourceAudioTrackSettingsForClip={getSourceAudioTrackSettingsForClip}
showSourceAudioTrack={showSourceAudioTrack}
liveSpanPreviewById={liveSpanPreviewById}
liveHiddenItemIds={liveHiddenItemIds}
direction={direction}
canShowGhostZoom={canShowGhostZoom}
ghostStartMs={ghostStartMs}
@@ -4,6 +4,11 @@ import type { AudioPeaksData } from "../../core/timelineTypes";
interface AudioWaveformProps {
peaks: AudioPeaksData;
segmentStartMs?: number;
segmentEndMs?: number;
gain?: number;
normalize?: boolean;
className?: string;
}
/**
@@ -11,10 +16,18 @@ interface AudioWaveformProps {
* Automatically syncs with the timeline's visible range so the waveform
* scrolls and zooms together with the clip items above it.
*/
function AudioWaveformComponent({ peaks }: AudioWaveformProps) {
function AudioWaveformComponent({
peaks,
segmentStartMs,
segmentEndMs,
gain = 1,
normalize = false,
className,
}: AudioWaveformProps) {
const canvasRef = useRef<HTMLCanvasElement>(null);
const { range } = useTimelineContext();
const [resizeKey, setResizeKey] = useState(0);
const lastDrawAtRef = useRef(0);
// Bump resizeKey when the canvas element changes size.
const observerRef = useRef<ResizeObserver | null>(null);
@@ -34,55 +47,77 @@ function AudioWaveformComponent({ peaks }: AudioWaveformProps) {
useEffect(() => {
const canvas = canvasRef.current;
if (!canvas) return;
let rafId = 0;
const ctx = canvas.getContext("2d");
if (!ctx) return;
const draw = () => {
const now = performance.now();
if (now - lastDrawAtRef.current < 33) {
rafId = requestAnimationFrame(draw);
return;
}
lastDrawAtRef.current = now;
const rect = canvas.getBoundingClientRect();
const dpr = window.devicePixelRatio || 1;
const width = Math.round(rect.width * dpr);
const height = Math.round(rect.height * dpr);
const ctx = canvas.getContext("2d");
if (!ctx) return;
if (width === 0 || height === 0) return;
const rect = canvas.getBoundingClientRect();
const dpr = window.devicePixelRatio || 1;
const width = Math.round(rect.width * dpr);
const height = Math.round(rect.height * dpr);
canvas.width = width;
canvas.height = height;
if (width === 0 || height === 0) return;
ctx.clearRect(0, 0, width, height);
canvas.width = width;
canvas.height = height;
const { peaks: peakData, durationMs } = peaks;
if (durationMs <= 0 || peakData.length === 0) return;
ctx.clearRect(0, 0, width, height);
const visibleStartMs = range.start;
const visibleEndMs = range.end;
const visibleDurationMs = visibleEndMs - visibleStartMs;
if (visibleDurationMs <= 0) return;
const { peaks: peakData, durationMs } = peaks;
if (durationMs <= 0 || peakData.length === 0) return;
const midY = height / 2;
const rawVisibleStartMs = segmentStartMs ?? range.start;
const rawVisibleEndMs = segmentEndMs ?? range.end;
const msPerBin = durationMs / peakData.length;
const visibleStartMs =
msPerBin > 0 ? Math.round(rawVisibleStartMs / msPerBin) * msPerBin : rawVisibleStartMs;
const visibleEndMs =
msPerBin > 0 ? Math.round(rawVisibleEndMs / msPerBin) * msPerBin : rawVisibleEndMs;
const visibleDurationMs = visibleEndMs - visibleStartMs;
if (visibleDurationMs <= 0) return;
ctx.beginPath();
for (let px = 0; px < width; px++) {
const t = visibleStartMs + (px / width) * visibleDurationMs;
const binIndex = Math.min(
peakData.length - 1,
Math.max(0, Math.floor((t / durationMs) * peakData.length)),
);
const amplitude = peakData[binIndex];
const barHeight = amplitude * midY * 0.85;
const midY = height / 2;
ctx.moveTo(px, midY - barHeight);
ctx.lineTo(px, midY + barHeight);
}
ctx.beginPath();
for (let px = 0; px < width; px++) {
const t = visibleStartMs + (px / width) * visibleDurationMs;
const exactIndex = Math.max(
0,
Math.min(peakData.length - 1, (t / durationMs) * (peakData.length - 1)),
);
const leftIndex = Math.floor(exactIndex);
const rightIndex = Math.min(peakData.length - 1, leftIndex + 1);
const mix = exactIndex - leftIndex;
let amplitude = peakData[leftIndex] * (1 - mix) + peakData[rightIndex] * mix;
if (normalize) amplitude = Math.sqrt(Math.max(0, amplitude));
amplitude = Math.max(0, Math.min(1, amplitude * gain));
const barHeight = amplitude * midY * 0.85;
ctx.strokeStyle = "rgba(255, 255, 255, 0.55)";
ctx.lineWidth = dpr;
ctx.stroke();
}, [peaks, range.start, range.end, resizeKey]);
ctx.moveTo(px, midY - barHeight);
ctx.lineTo(px, midY + barHeight);
}
ctx.strokeStyle = "rgba(255, 255, 255, 0.55)";
ctx.lineWidth = dpr;
ctx.stroke();
};
rafId = requestAnimationFrame(draw);
return () => cancelAnimationFrame(rafId);
}, [gain, normalize, peaks, range.start, range.end, resizeKey, segmentStartMs, segmentEndMs]);
return (
<canvas
ref={setCanvasRef}
className="absolute inset-0 w-full h-full pointer-events-none"
className={className ?? "absolute inset-0 w-full h-full pointer-events-none"}
style={{ display: "block" }}
/>
);
@@ -29,6 +29,7 @@ interface TimelineWrapperProps {
onItemSpanChange: (id: string, span: Span, rowId?: string) => void;
resolveTargetRowId?: (id: string, proposedRowId: string) => string;
allRegionSpans?: TimelineRegionSpan[];
onLiveSpanPreviewChange?: (id: string, span: Span | null) => void;
}
export default function TimelineWrapper({
@@ -43,6 +44,7 @@ export default function TimelineWrapper({
onItemSpanChange,
resolveTargetRowId,
allRegionSpans = [],
onLiveSpanPreviewChange,
}: TimelineWrapperProps) {
const totalMs = Math.max(0, Math.round(videoDuration * 1000));
@@ -144,8 +146,12 @@ export default function TimelineWrapper({
? (event.activatorEvent as PointerEvent).clientX + (event.delta?.x ?? 0)
: undefined;
if (span) showTooltip(span, screenX);
const moved = Math.hypot(event.delta?.x ?? 0, event.delta?.y ?? 0) > 0.01;
if (moved) {
onLiveSpanPreviewChange?.(event.active.id as string, span ?? null);
}
},
[showTooltip],
[onLiveSpanPreviewChange, showTooltip],
);
const onResizeMove = useCallback(
@@ -156,8 +162,9 @@ export default function TimelineWrapper({
? (event.activatorEvent as PointerEvent).clientX + (event.delta?.x ?? 0)
: undefined;
if (span) showTooltip(span, screenX);
onLiveSpanPreviewChange?.(event.active.id as string, span ?? null);
},
[showTooltip],
[onLiveSpanPreviewChange, showTooltip],
);
const hideTooltip = useCallback(() => showTooltip(null), [showTooltip]);
@@ -166,16 +173,18 @@ export default function TimelineWrapper({
(event: ResizeEndEvent) => {
hideTooltip();
onResizeEnd(event);
onLiveSpanPreviewChange?.(event.active.id as string, null);
},
[hideTooltip, onResizeEnd],
[hideTooltip, onLiveSpanPreviewChange, onResizeEnd],
);
const onDragEndWithTooltip = useCallback(
(event: DragEndEvent) => {
hideTooltip();
onDragEnd(event);
onLiveSpanPreviewChange?.(event.active.id as string, null);
},
[hideTooltip, onDragEnd],
[hideTooltip, onDragEnd, onLiveSpanPreviewChange],
);
const handleRangeChange = useCallback(
@@ -2,8 +2,10 @@ export const ZOOM_ROW_ID = "row-zoom";
export const CLIP_ROW_ID = "row-clip";
export const ANNOTATION_ROW_ID = "row-annotation";
export const AUDIO_ROW_ID = "row-audio";
export const SOURCE_AUDIO_ROW_ID = "row-source-audio";
export const ANNOTATION_ROW_PREFIX = `${ANNOTATION_ROW_ID}-`;
export const AUDIO_ROW_PREFIX = `${AUDIO_ROW_ID}-`;
export const FALLBACK_RANGE_MS = 1000;
export const TARGET_MARKER_COUNT = 12;
export const WAVEFORM_DEFAULT_PEAK_COUNT = 2048;
@@ -31,10 +31,16 @@ export interface TimelineRenderItem {
id: string;
rowId: string;
span: Span;
sourceSpan?: Span;
label: string;
audioPath?: string;
audioGain?: number;
audioNormalize?: boolean;
zoomDepth?: number;
zoomMode?: ZoomMode;
speedValue?: number;
showSourceAudio?: boolean;
muted?: boolean;
variant: "zoom" | "trim" | "clip" | "annotation" | "speed" | "audio";
}
@@ -1,81 +1,100 @@
import { useEffect, useRef, useState } from "react";
import { resolveMediaResourceUrl } from "@/lib/exporter/localMediaSource";
import { fromFileUrl } from "../../projectPersistence";
import { waveformGenerator } from "../../audio/waveform/WaveformGenerator";
import { WAVEFORM_DEFAULT_PEAK_COUNT } from "../core/constants";
import type { AudioPeaksData } from "../core/timelineTypes";
/** Number of peak bins to produce — enough for smooth display at any zoom. */
const TARGET_PEAK_COUNT = 2048;
function buildSidecarAudioCandidates(sourcePath: string): string[] {
const normalized = sourcePath.replace(/\\/g, "/");
const lastSlash = normalized.lastIndexOf("/");
const dir = lastSlash >= 0 ? normalized.slice(0, lastSlash + 1) : "";
const fileName = lastSlash >= 0 ? normalized.slice(lastSlash + 1) : normalized;
const dotIndex = fileName.lastIndexOf(".");
const baseName = dotIndex > 0 ? fileName.slice(0, dotIndex) : fileName;
/**
* Decode audio from a media file URL and produce a fixed-length array of peak
* amplitudes suitable for waveform visualisation.
*
* Returns `null` while loading or if the file has no decodeable audio.
*/
export function useTimelineAudioPeaks(fileUrl: string | null | undefined): AudioPeaksData | null {
return [
`${dir}${baseName}.system.wav`,
`${dir}${baseName}.mic.wav`,
`${dir}${baseName}.system.m4a`,
`${dir}${baseName}.mic.m4a`,
];
}
function extractLocalPathFromMediaServerUrl(input: string): string | null {
try {
const url = new URL(input);
const isLocalMediaServer =
(url.protocol === "http:" || url.protocol === "https:") &&
(url.hostname === "127.0.0.1" || url.hostname === "localhost") &&
url.pathname === "/video";
if (!isLocalMediaServer) return null;
return url.searchParams.get("path");
} catch {
return null;
}
}
interface TimelineAudioPeaksOptions {
enableSourceSidecarFallback?: boolean;
peakCount?: number;
}
export function useTimelineAudioPeaks(
mediaResource: string | null | undefined,
options: TimelineAudioPeaksOptions = {},
): AudioPeaksData | null {
const [data, setData] = useState<AudioPeaksData | null>(null);
const urlRef = useRef(fileUrl);
const sourceRef = useRef(mediaResource);
const enableSourceSidecarFallback = options.enableSourceSidecarFallback ?? false;
const peakCount = options.peakCount ?? WAVEFORM_DEFAULT_PEAK_COUNT;
useEffect(() => {
urlRef.current = fileUrl;
sourceRef.current = mediaResource;
setData(null);
if (!fileUrl) {
return;
}
if (!mediaResource) return;
let cancelled = false;
(async () => {
const run = async () => {
const tryGenerate = async (resource: string): Promise<AudioPeaksData> => {
const resolvedUrl = await resolveMediaResourceUrl(resource);
return waveformGenerator.generate(resolvedUrl, peakCount);
};
try {
const response = await fetch(fileUrl);
if (cancelled) return;
const arrayBuffer = await response.arrayBuffer();
if (cancelled) return;
const audioCtx = new OfflineAudioContext(1, 1, 44100);
const decoded = await audioCtx.decodeAudioData(arrayBuffer);
if (cancelled) return;
const channelData = decoded.getChannelData(0);
const durationMs = decoded.duration * 1000;
const binSize = Math.max(1, Math.floor(channelData.length / TARGET_PEAK_COUNT));
const peakCount = Math.ceil(channelData.length / binSize);
const peaks = new Float32Array(peakCount);
for (let i = 0; i < peakCount; i++) {
const start = i * binSize;
const end = Math.min(start + binSize, channelData.length);
let max = 0;
for (let j = start; j < end; j++) {
const abs = Math.abs(channelData[j]);
if (abs > max) max = abs;
}
peaks[i] = max;
}
// Normalise to 0–1 range.
let globalMax = 0;
for (let i = 0; i < peaks.length; i++) {
if (peaks[i] > globalMax) globalMax = peaks[i];
}
if (globalMax > 0) {
for (let i = 0; i < peaks.length; i++) {
peaks[i] /= globalMax;
}
}
if (!cancelled && urlRef.current === fileUrl) {
setData({ peaks, durationMs });
}
const result = await tryGenerate(mediaResource);
if (!cancelled && sourceRef.current === mediaResource) setData(result);
return;
} catch {
// File has no audio or decoding failed — leave as null.
// fallthrough
}
})();
if (!enableSourceSidecarFallback) return;
const localPathFromServer = extractLocalPathFromMediaServerUrl(mediaResource);
const localSourcePath =
localPathFromServer ||
(/^file:\/\//i.test(mediaResource) ? fromFileUrl(mediaResource) : mediaResource);
if (!localSourcePath) return;
for (const candidate of buildSidecarAudioCandidates(localSourcePath)) {
try {
const result = await tryGenerate(candidate);
if (!cancelled && sourceRef.current === mediaResource) setData(result);
return;
} catch {
// try next
}
}
};
void run();
return () => {
cancelled = true;
};
}, [fileUrl]);
}, [mediaResource, enableSourceSidecarFallback, peakCount]);
return data;
}
@@ -47,13 +47,22 @@ export function buildTimelineItems(params: {
variant: "zoom",
}));
const clips: TimelineRenderItem[] = clipRegions.map((region, index) => ({
id: region.id,
rowId: CLIP_ROW_ID,
span: { start: region.startMs, end: region.endMs },
label: `Clip ${index + 1}`,
variant: "clip",
}));
const clips: TimelineRenderItem[] = clipRegions.map((region, index) => {
const displayDurationMs = Math.max(0, region.endMs - region.startMs);
const speed = Number.isFinite(region.speed) && region.speed > 0 ? region.speed : 1;
const sourceEndMs = region.startMs + displayDurationMs * speed;
return {
id: region.id,
rowId: CLIP_ROW_ID,
span: { start: region.startMs, end: region.endMs },
sourceSpan: { start: region.startMs, end: sourceEndMs },
label: `Clip ${index + 1}`,
showSourceAudio: region.showSourceAudio,
muted: Boolean(region.muted),
variant: "clip",
};
});
const annotations: TimelineRenderItem[] = annotationRegions.map((region) => ({
id: region.id,
@@ -68,6 +77,9 @@ export function buildTimelineItems(params: {
rowId: getAudioTrackRowId(region.trackIndex ?? 0),
span: { start: region.startMs, end: region.endMs },
label: getAudioLabel(region),
audioPath: region.audioPath,
audioGain: region.volume,
audioNormalize: Boolean(region.normalize),
variant: "audio",
}));
+4
View File
@@ -64,6 +64,7 @@ export type EditorEffectSection =
| "crop"
| "extensions"
| "clip"
| "audio"
| `ext:${string}`;
export type ZoomTransitionEasing = "recordly" | "glide" | "smooth" | "snappy" | "linear";
@@ -169,6 +170,7 @@ export interface ClipRegion {
endMs: number;
speed: number;
muted?: boolean;
showSourceAudio?: boolean;
}
export function getClipSourceEndMs(clip: ClipRegion): number {
@@ -464,6 +466,7 @@ export const DEFAULT_PADDING: Padding = {
right: 20,
linked: true,
};
export type { SourceAudioTrackSetting, SourceAudioTrackSettings } from "@/components/video-editor/audio/audioTypes";
export interface AudioRegion {
id: string;
@@ -471,6 +474,7 @@ export interface AudioRegion {
endMs: number;
audioPath: string;
volume: number;
normalize?: boolean;
trackIndex?: number;
}
+15 -1
View File
@@ -19,7 +19,10 @@
},
"clip": {
"title": "Clip",
"muteAudio": "Mute Audio",
"mute": "Mute",
"mutedState": "Audio is muted",
"unmutedState": "Audio is playing",
"separateClipFromAudio": "Separate clip from audio",
"delete": "Delete Clip"
},
"effects": {
@@ -201,5 +204,16 @@
"exportVideo": "Export {{format}}",
"reportBug": "Report Bug",
"starOnGithub": "Star on GitHub"
},
"audio": {
"title": "Audio",
"volumeTitle": "Audio",
"volume": "Volume",
"normalize": "Normalize",
"sourceTracksTitle": "Clip Source Audio",
"systemLabel": "Source System",
"micLabel": "Source Mic",
"mixedLabel": "Source",
"deleteRegion": "Delete Audio"
}
}
+15 -1
View File
@@ -19,7 +19,10 @@
},
"clip": {
"title": "Clip",
"muteAudio": "Silenciar audio",
"mute": "Silenciar",
"mutedState": "El audio está silenciado",
"unmutedState": "El audio se está reproduciendo",
"separateClipFromAudio": "Separar clip del audio",
"delete": "Eliminar clip"
},
"effects": {
@@ -181,5 +184,16 @@
"exportVideo": "Exportar {{format}}",
"reportBug": "Reportar error",
"starOnGithub": "Estrella en GitHub"
},
"audio": {
"title": "Audio",
"volumeTitle": "Audio",
"volume": "Volumen",
"normalize": "Normalizar",
"sourceTracksTitle": "Audio fuente del clip",
"systemLabel": "Sonido del sistema",
"micLabel": "Micrófono",
"mixedLabel": "Fuente",
"deleteRegion": "Eliminar audio"
}
}
+15 -1
View File
@@ -19,7 +19,10 @@
},
"clip": {
"title": "Clip",
"muteAudio": "Couper le son",
"mute": "Sourdine",
"mutedState": "Le son est coupé",
"unmutedState": "Le son est activé",
"separateClipFromAudio": "Séparer le clip de l'audio",
"delete": "Supprimer le clip"
},
"effects": {
@@ -181,5 +184,16 @@
"exportVideo": "Exporter en {{format}}",
"reportBug": "Signaler un bug",
"starOnGithub": "Mettre une étoile sur GitHub"
},
"audio": {
"title": "Audio",
"volumeTitle": "Audio",
"volume": "Volume",
"normalize": "Normaliser",
"sourceTracksTitle": "Source audio du clip",
"systemLabel": "Son système",
"micLabel": "Microphone",
"mixedLabel": "Source",
"deleteRegion": "Supprimer la zone audio"
}
}
+15 -1
View File
@@ -19,7 +19,10 @@
},
"clip": {
"title": "클립",
"muteAudio": "오디오 음소거",
"mute": "음소거",
"mutedState": "오디오가 음소거됨",
"unmutedState": "오디오가 재생 중",
"separateClipFromAudio": "클립에서 오디오 분리",
"delete": "클립 삭제"
},
"effects": {
@@ -181,5 +184,16 @@
"exportVideo": "{{format}} 내보내기",
"reportBug": "버그 신고",
"starOnGithub": "GitHub에서 별표 주기"
},
"audio": {
"title": "오디오",
"volumeTitle": "오디오",
"volume": "볼륨",
"normalize": "정규화",
"sourceTracksTitle": "클립 소스 오디오",
"systemLabel": "시스템 소스",
"micLabel": "마이크 소스",
"mixedLabel": "소스",
"deleteRegion": "오디오 삭제"
}
}
+15 -1
View File
@@ -19,7 +19,10 @@
},
"clip": {
"title": "Clip",
"muteAudio": "Audio dempen",
"mute": "Dempen",
"mutedState": "Audio is gedempt",
"unmutedState": "Audio wordt afgespeeld",
"separateClipFromAudio": "Clip van audio scheiden",
"delete": "Clip verwijderen"
},
"effects": {
@@ -181,5 +184,16 @@
"exportVideo": "{{format}} exporteren",
"reportBug": "Bug melden",
"starOnGithub": "Ster op GitHub"
},
"audio": {
"title": "Audio",
"volumeTitle": "Audio",
"volume": "Volume",
"normalize": "Normaliseren",
"sourceTracksTitle": "Bronaudio clip",
"systemLabel": "Systeemaudio",
"micLabel": "Microfoon",
"mixedLabel": "Bron",
"deleteRegion": "Audio verwijderen"
}
}
+15 -1
View File
@@ -19,7 +19,10 @@
},
"clip": {
"title": "Clipe",
"muteAudio": "Silenciar áudio",
"mute": "Mudo",
"mutedState": "O áudio está mudo",
"unmutedState": "O áudio está tocando",
"separateClipFromAudio": "Separar áudio do clipe",
"delete": "Excluir clipe"
},
"effects": {
@@ -181,5 +184,16 @@
"exportVideo": "Exportar {{format}}",
"reportBug": "Reportar bug",
"starOnGithub": "Dar estrela no GitHub"
},
"audio": {
"title": "Áudio",
"volumeTitle": "Áudio",
"volume": "Volume",
"normalize": "Normalizar",
"sourceTracksTitle": "Áudio original do clipe",
"systemLabel": "Sistema",
"micLabel": "Microfone",
"mixedLabel": "Fonte",
"deleteRegion": "Excluir áudio"
}
}
+16 -2
View File
@@ -19,8 +19,11 @@
},
"clip": {
"title": "Клип",
"muteAudio": "Выключить звук",
"delete": "Удалить"
"mute": "Без звука",
"mutedState": "Звук выключен",
"unmutedState": "Звук включен",
"separateClipFromAudio": "Отделить аудио от клипа",
"delete": "Удалить клип"
},
"effects": {
"title": "Эффекты",
@@ -201,5 +204,16 @@
"exportVideo": "Экспортировать {{format}}",
"reportBug": "Сообщить об ошибке",
"starOnGithub": "Оценить на GitHub"
},
"audio": {
"title": "Аудио",
"volumeTitle": "Аудио",
"volume": "Громкость",
"normalize": "Нормализовать",
"sourceTracksTitle": "Исходное аудио клипа",
"systemLabel": "Системный звук",
"micLabel": "Микрофон",
"mixedLabel": "Источник",
"deleteRegion": "Удалить аудио"
}
}
+16 -2
View File
@@ -19,7 +19,10 @@
},
"clip": {
"title": "片段",
"muteAudio": "静音音频",
"mute": "静音",
"mutedState": "音频已静音",
"unmutedState": "音频正在播放",
"separateClipFromAudio": "将剪辑与音频分离",
"delete": "删除片段"
},
"effects": {
@@ -195,6 +198,17 @@
"saveProject": "保存项目",
"exportVideo": "导出{{format}}",
"reportBug": "报告问题",
"starOnGithub": "在 GitHub 上加星"
"starOnGithub": "在 GitHub 上点赞"
},
"audio": {
"title": "音频",
"volumeTitle": "音频",
"volume": "音量",
"normalize": "标准化",
"sourceTracksTitle": "剪辑源音频",
"systemLabel": "系统声音",
"micLabel": "麦克风",
"mixedLabel": "来源",
"deleteRegion": "删除音频"
}
}
+15 -1
View File
@@ -19,7 +19,10 @@
},
"clip": {
"title": "片段",
"muteAudio": "靜音",
"mute": "靜音",
"mutedState": "音訊已靜音",
"unmutedState": "音訊正在播放",
"separateClipFromAudio": "將片段與音訊分離",
"delete": "刪除片段"
},
"effects": {
@@ -181,5 +184,16 @@
"exportVideo": "匯出 {{format}}",
"reportBug": "回報錯誤",
"starOnGithub": "在 GitHub 按讚"
},
"audio": {
"title": "音訊",
"volumeTitle": "音訊",
"volume": "音量",
"normalize": "正規化",
"sourceTracksTitle": "剪輯源音訊",
"systemLabel": "系統源",
"micLabel": "麥克風源",
"mixedLabel": "源",
"deleteRegion": "刪除音訊"
}
}
+195 -31
View File
@@ -3,12 +3,18 @@ import type {
AudioRegion,
ClipRegion,
SpeedRegion,
SourceAudioTrackSettings,
TrimRegion,
} from "@/components/video-editor/types";
import {
buildResolvedAudioPlan,
SourceTrackId,
} from "@/lib/exporter/audioRoutingEngine";
import { estimateCompanionAudioStartDelaySeconds } from "@/lib/mediaTiming";
import { resolveMediaElementSource } from "./localMediaSource";
import type { VideoMuxer } from "./muxer";
import { resolveSourceAudioFallbackPaths } from "./sourceAudioFallback";
import { resolveSourceTrackRoutingPolicy } from "./sourceTrackRoutingPolicy";
import { SOURCE_AUDIO_NORMALIZE_GAIN } from "@/components/video-editor/audio/audioTypes";
const AUDIO_BITRATE = 128_000;
const DECODE_BACKPRESSURE_LIMIT = 20;
@@ -19,6 +25,60 @@ const OFFLINE_AUDIO_SAMPLE_RATE = 48_000;
const OFFLINE_ENCODE_CHUNK_FRAMES = 1024;
const OFFLINE_CHUNK_DURATION_SEC = 30;
function resolveSourceTrackGain(
sourceAudioTrackSettings: SourceAudioTrackSettings | undefined,
trackId: "mic" | "system" | "mixed",
) {
const settings = sourceAudioTrackSettings?.[trackId];
if (!settings) {
return 1;
}
const normalizeGain = settings.normalize ? SOURCE_AUDIO_NORMALIZE_GAIN : 1;
return Math.max(0, Math.min(2, settings.volume * normalizeGain));
}
export function getSourceTrackIdFromPath(audioPath: string): SourceTrackId {
const normalized = audioPath.toLowerCase();
// Check for common patterns like .mic., -mic., mic.mp4, etc.
if (
normalized.includes(".mic.") ||
normalized.includes("-mic.") ||
normalized.includes("_mic_") ||
normalized.includes("/mic.") ||
normalized.includes("\\mic.") ||
normalized.endsWith("mic.mp4") ||
normalized.endsWith("mic.m4a") ||
normalized.endsWith("mic.wav")
) {
return "mic";
}
if (
normalized.includes(".system.") ||
normalized.includes("-system.") ||
normalized.includes("_system_") ||
normalized.includes("/system.") ||
normalized.includes("\\system.") ||
normalized.endsWith("system.mp4") ||
normalized.endsWith("system.m4a") ||
normalized.endsWith("system.wav")
) {
return "system";
}
return "mixed";
}
export function hasNonDefaultSourceTrackSettings(
sourceAudioTrackSettings?: SourceAudioTrackSettings,
) {
if (!sourceAudioTrackSettings) {
return false;
}
return Object.values(sourceAudioTrackSettings).some(
(settings) =>
Math.abs((settings?.volume ?? 1) - 1) > 0.0005 || Boolean(settings?.normalize),
);
}
interface TimelineSlice {
sourceStartMs: number;
sourceEndMs: number;
@@ -26,9 +86,10 @@ interface TimelineSlice {
}
interface PreparedOfflineRender {
mainBuffer: AudioBuffer | null;
companionEntries: Array<{ buffer: AudioBuffer; startDelaySec: number }>;
mainBufferEntry: { buffer: AudioBuffer; gain: number } | null;
companionEntries: Array<{ buffer: AudioBuffer; startDelaySec: number; gain: number }>;
regionEntries: Array<{ buffer: AudioBuffer; region: AudioRegion }>;
mutedSourceOutputRangesSec: Array<{ startSec: number; endSec: number }>;
slices: TimelineSlice[];
outputDurationMs: number;
numChannels: number;
@@ -144,6 +205,8 @@ export class AudioProcessor {
audioRegions?: AudioRegion[],
sourceAudioFallbackPaths?: string[],
sourceAudioFallbackStartDelayMsByPath?: Record<string, number>,
sourceAudioTrackSettings?: SourceAudioTrackSettings,
clipRegions?: ClipRegion[],
): Promise<void> {
const sortedTrims = trimRegions
? [...trimRegions].sort((a, b) => a.startMs - b.startMs)
@@ -161,23 +224,25 @@ export class AudioProcessor {
(audioPath) => typeof audioPath === "string" && audioPath.trim().length > 0,
)
: [];
const { hasEmbeddedSourceAudio, externalAudioPaths } = resolveSourceAudioFallbackPaths(
const routingPolicy = resolveSourceTrackRoutingPolicy(
videoUrl,
sortedSourceAudioFallbackPaths,
);
const hasTimedCompanionAudio = externalAudioPaths.some(
const hasTimedCompanionAudio = routingPolicy.playbackPaths.some(
(audioPath) => (sourceAudioFallbackStartDelayMsByPath?.[audioPath] ?? 0) > 0,
);
const needsSourceAudioMixing =
externalAudioPaths.length > 1 ||
(hasEmbeddedSourceAudio && externalAudioPaths.length > 0) ||
routingPolicy.playbackPaths.length > 1 ||
(routingPolicy.hasEmbeddedSourceAudio && routingPolicy.playbackPaths.length > 0) ||
hasTimedCompanionAudio;
// When speed edits, audio regions, or multiple audio sources need mixing, use offline AudioContext pipeline.
if (
sortedSpeedRegions.length > 0 ||
sortedAudioRegions.length > 0 ||
needsSourceAudioMixing
needsSourceAudioMixing ||
hasNonDefaultSourceTrackSettings(sourceAudioTrackSettings) ||
(clipRegions ?? []).some((clip) => Boolean(clip.muted))
) {
await this.renderAndMuxOfflineAudio(
videoUrl,
@@ -186,14 +251,16 @@ export class AudioProcessor {
sortedAudioRegions,
sortedSourceAudioFallbackPaths,
sourceAudioFallbackStartDelayMsByPath,
sourceAudioTrackSettings,
clipRegions,
muxer,
);
return;
}
// Single sidecar audio with no speed/audio edits: demux directly (skips slow real-time rendering).
if (!hasEmbeddedSourceAudio && externalAudioPaths.length === 1) {
const sidecarDemuxer = await this.loadAudioFileDemuxer(externalAudioPaths[0]);
if (!routingPolicy.hasEmbeddedSourceAudio && routingPolicy.playbackPaths.length === 1) {
const sidecarDemuxer = await this.loadAudioFileDemuxer(routingPolicy.playbackPaths[0]);
if (sidecarDemuxer) {
try {
await this.processTrimOnlyAudio(sidecarDemuxer, muxer, sortedTrims);
@@ -215,8 +282,10 @@ export class AudioProcessor {
sortedTrims,
[],
[],
externalAudioPaths,
routingPolicy.playbackPaths,
sourceAudioFallbackStartDelayMsByPath,
sourceAudioTrackSettings,
clipRegions,
muxer,
);
return;
@@ -263,6 +332,8 @@ export class AudioProcessor {
audioRegions?: AudioRegion[],
sourceAudioFallbackPaths?: string[],
sourceAudioFallbackStartDelayMsByPath?: Record<string, number>,
sourceAudioTrackSettings?: SourceAudioTrackSettings,
clipRegions?: ClipRegion[],
): Promise<Blob> {
const sortedTrims = trimRegions
? [...trimRegions].sort((a, b) => a.startMs - b.startMs)
@@ -288,6 +359,8 @@ export class AudioProcessor {
sortedAudioRegions,
sortedSourceAudioFallbackPaths,
sourceAudioFallbackStartDelayMsByPath,
sourceAudioTrackSettings,
clipRegions,
);
return this.renderToWavBlobChunked(prepared);
}
@@ -563,6 +636,8 @@ export class AudioProcessor {
audioRegions: AudioRegion[],
sourceAudioFallbackPaths: string[],
sourceAudioFallbackStartDelayMsByPath: Record<string, number> | undefined,
sourceAudioTrackSettings: SourceAudioTrackSettings | undefined,
clipRegions: ClipRegion[] | undefined,
muxer: VideoMuxer,
): Promise<void> {
const prepared = await this.prepareOfflineRender(
@@ -572,6 +647,8 @@ export class AudioProcessor {
audioRegions,
sourceAudioFallbackPaths,
sourceAudioFallbackStartDelayMsByPath,
sourceAudioTrackSettings,
clipRegions,
);
if (this.cancelled) return;
await this.renderAndEncodeChunked(prepared, muxer);
@@ -584,31 +661,59 @@ export class AudioProcessor {
audioRegions: AudioRegion[],
sourceAudioFallbackPaths: string[],
sourceAudioFallbackStartDelayMsByPath?: Record<string, number>,
sourceAudioTrackSettings?: SourceAudioTrackSettings,
clipRegions?: ClipRegion[],
): Promise<PreparedOfflineRender> {
if (this.cancelled) throw new Error("Export cancelled");
this.onProgress?.(0);
const { externalAudioPaths } = resolveSourceAudioFallbackPaths(
videoUrl,
const resolvedPlan = buildResolvedAudioPlan({
videoResource: videoUrl,
sourceAudioFallbackPaths,
);
audioRegions,
sourceTrackGainById: {
mic: resolveSourceTrackGain(sourceAudioTrackSettings, "mic"),
system: resolveSourceTrackGain(sourceAudioTrackSettings, "system"),
mixed: resolveSourceTrackGain(sourceAudioTrackSettings, "mixed"),
},
embeddedGain: Math.max(
0,
Math.min(
2,
sourceAudioTrackSettings?.mixed
? resolveSourceTrackGain(sourceAudioTrackSettings, "mixed")
: sourceAudioTrackSettings?.system
? resolveSourceTrackGain(sourceAudioTrackSettings, "system")
: 1,
),
),
});
// Decode embedded source audio separately from companion sidecars.
const mainBuffer = await this.decodeAudioFromUrl(videoUrl);
const mainBuffer = resolvedPlan.includeEmbeddedInExport
? await this.decodeAudioFromUrl(videoUrl)
: null;
const mainBufferGain = resolveSourceTrackGain(sourceAudioTrackSettings, "mixed");
const mainBufferEntry = mainBuffer ? { buffer: mainBuffer, gain: mainBufferGain } : null;
if (this.cancelled) throw new Error("Export cancelled");
// Decode companion / sidecar audio files
const companionEntries: Array<{ buffer: AudioBuffer; startDelaySec: number }> = [];
const companionEntries: Array<{ buffer: AudioBuffer; startDelaySec: number; gain: number }> =
[];
const refDuration =
mainBuffer?.duration ??
(externalAudioPaths.length > 0 ? await this.getMediaDurationSec(videoUrl) : 0);
for (const audioPath of externalAudioPaths) {
(resolvedPlan.playbackPaths.length > 0 ? await this.getMediaDurationSec(videoUrl) : 0);
for (const audioPath of resolvedPlan.playbackPaths) {
if (this.cancelled) throw new Error("Export cancelled");
const buffer = await this.decodeAudioFromUrl(audioPath);
if (!buffer) continue;
companionEntries.push({
buffer,
gain: resolveSourceTrackGain(
sourceAudioTrackSettings,
getSourceTrackIdFromPath(audioPath),
),
startDelaySec: estimateCompanionAudioStartDelaySeconds(
refDuration,
buffer.duration,
@@ -629,15 +734,15 @@ export class AudioProcessor {
this.onProgress?.(0.2);
// Determine source duration for timeline calculation
const primaryBuffer = mainBuffer ?? companionEntries[0]?.buffer ?? null;
const primaryBuffer = mainBufferEntry?.buffer ?? companionEntries[0]?.buffer ?? null;
if (!primaryBuffer && regionEntries.length === 0) {
throw new Error("No decodable audio sources found");
}
let sourceDurationSec: number;
if (mainBuffer) {
sourceDurationSec = mainBuffer.duration;
} else if (externalAudioPaths.length > 0 || regionEntries.length > 0) {
if (mainBufferEntry?.buffer) {
sourceDurationSec = mainBufferEntry.buffer.duration;
} else if (resolvedPlan.playbackPaths.length > 0 || regionEntries.length > 0) {
sourceDurationSec = await this.getMediaDurationSec(videoUrl);
} else {
sourceDurationSec = primaryBuffer?.duration ?? 0;
@@ -659,11 +764,24 @@ export class AudioProcessor {
}
const numChannels = Math.min(primaryBuffer?.numberOfChannels ?? 2, 2);
const mutedSourceOutputRangesSec = (clipRegions ?? [])
.filter(
(clip) =>
Boolean(clip.muted) &&
Number.isFinite(clip.startMs) &&
Number.isFinite(clip.endMs) &&
clip.endMs > clip.startMs,
)
.map((clip) => ({
startSec: Math.max(0, clip.startMs / 1000),
endSec: Math.max(0, clip.endMs / 1000),
}));
return {
mainBuffer,
mainBufferEntry,
companionEntries,
regionEntries,
mutedSourceOutputRangesSec,
slices,
outputDurationMs,
numChannels,
@@ -788,14 +906,16 @@ export class AudioProcessor {
);
// Schedule main audio
if (prepared.mainBuffer) {
if (prepared.mainBufferEntry) {
this.scheduleBufferThroughTimeline(
offlineCtx,
prepared.mainBuffer,
prepared.mainBufferEntry.buffer,
slices,
0,
prepared.mainBufferEntry.gain,
outputOffsetSec,
chunkSec,
prepared.mutedSourceOutputRangesSec,
);
}
@@ -806,8 +926,10 @@ export class AudioProcessor {
entry.buffer,
slices,
entry.startDelaySec,
entry.gain,
outputOffsetSec,
chunkSec,
prepared.mutedSourceOutputRangesSec,
);
}
@@ -865,7 +987,8 @@ export class AudioProcessor {
if (duration <= 0.001) return;
const gainNode = ctx.createGain();
gainNode.gain.value = Math.max(0, Math.min(1, region.volume));
const normalizeGain = region.normalize ? SOURCE_AUDIO_NORMALIZE_GAIN : 1;
gainNode.gain.value = Math.max(0, Math.min(1, region.volume * normalizeGain));
gainNode.connect(ctx.destination);
const source = ctx.createBufferSource();
@@ -1265,8 +1388,10 @@ export class AudioProcessor {
buffer: AudioBuffer,
slices: TimelineSlice[],
sourceStartDelaySec: number,
gain = 1,
chunkOutputStartSec = 0,
chunkDurationSec = Number.POSITIVE_INFINITY,
mutedOutputRangesSec: Array<{ startSec: number; endSec: number }> = [],
): void {
let outputOffsetSec = 0;
@@ -1330,12 +1455,51 @@ export class AudioProcessor {
continue;
}
const source = ctx.createBufferSource();
source.buffer = buffer;
source.playbackRate.value = slice.speed;
source.connect(ctx.destination);
const audibleRanges: Array<{ startSec: number; endSec: number }> = [
{
startSec: localOutputStartSec + chunkOutputStartSec,
endSec:
localOutputStartSec + chunkOutputStartSec + effectiveSourceDurationSec / slice.speed,
},
];
for (const mutedRange of mutedOutputRangesSec) {
for (let rangeIndex = audibleRanges.length - 1; rangeIndex >= 0; rangeIndex -= 1) {
const current = audibleRanges[rangeIndex];
const overlapStart = Math.max(current.startSec, mutedRange.startSec);
const overlapEnd = Math.min(current.endSec, mutedRange.endSec);
if (overlapEnd <= overlapStart) {
continue;
}
audibleRanges.splice(rangeIndex, 1);
if (current.startSec < overlapStart) {
audibleRanges.push({ startSec: current.startSec, endSec: overlapStart });
}
if (overlapEnd < current.endSec) {
audibleRanges.push({ startSec: overlapEnd, endSec: current.endSec });
}
}
}
source.start(localOutputStartSec, effectiveBufferStartSec, effectiveSourceDurationSec);
for (const audibleRange of audibleRanges) {
const audibleDurationSec = audibleRange.endSec - audibleRange.startSec;
if (audibleDurationSec <= 0.001) {
continue;
}
const source = ctx.createBufferSource();
const gainNode = ctx.createGain();
gainNode.gain.value = Math.max(0, Math.min(2, gain));
source.buffer = buffer;
source.playbackRate.value = slice.speed;
source.connect(gainNode);
gainNode.connect(ctx.destination);
const sourceOffsetSec =
effectiveBufferStartSec +
(audibleRange.startSec - (localOutputStartSec + chunkOutputStartSec)) * slice.speed;
const localStartSec = audibleRange.startSec - chunkOutputStartSec;
const sourceDurationSec = audibleDurationSec * slice.speed;
source.start(localStartSec, sourceOffsetSec, sourceDurationSec);
}
outputOffsetSec += sliceOutputDurationSec;
}
+129
View File
@@ -0,0 +1,129 @@
import type { AudioRegion } from "@/components/video-editor/types";
import { SOURCE_AUDIO_NORMALIZE_GAIN } from "@/components/video-editor/audio/audioTypes";
import { resolveSourceAudioFallbackPaths } from "./sourceAudioFallback";
export type SourceTrackId = "mic" | "system" | "mixed";
export type ResolvedAudioTrackKind = "user" | "system" | "mic" | "mixed" | "embedded";
export interface ResolvedAudioTrack {
id: string;
kind: ResolvedAudioTrackKind;
sourceRef: {
path: string;
startDelayMs: number;
};
gain: number;
timelineBinding: {
startMs: number;
endMs: number;
};
}
export interface ResolvedAudioPlan {
hasEmbeddedSourceAudio: boolean;
pathsByTrack: Partial<Record<SourceTrackId, string>>;
playbackPaths: string[];
muteEmbeddedPreview: boolean;
includeEmbeddedInExport: boolean;
tracks: ResolvedAudioTrack[];
masterGain: number;
}
export function getSourceTrackIdFromPath(audioPath: string): SourceTrackId {
const normalized = audioPath.toLowerCase();
if (normalized.includes(".mic.")) return "mic";
if (normalized.includes(".system.")) return "system";
return "mixed";
}
function clampGain(value: number, max: number) {
if (!Number.isFinite(value)) return 1;
return Math.max(0, Math.min(max, value));
}
export function buildResolvedAudioPlan(input: {
videoResource: string | null | undefined;
sourceAudioFallbackPaths: string[] | null | undefined;
audioRegions?: AudioRegion[];
sourceTrackGainById?: Partial<Record<SourceTrackId, number>>;
embeddedGain?: number;
masterGain?: number;
}): ResolvedAudioPlan {
const { hasEmbeddedSourceAudio, externalAudioPaths } = resolveSourceAudioFallbackPaths(
input.videoResource,
input.sourceAudioFallbackPaths,
);
const pathsByTrack: Partial<Record<SourceTrackId, string>> = {};
for (const path of externalAudioPaths) {
const trackId = getSourceTrackIdFromPath(path);
if (!pathsByTrack[trackId]) {
pathsByTrack[trackId] = path;
}
}
const hasDedicatedTracks = Boolean(pathsByTrack.system || pathsByTrack.mic);
const playbackPaths: string[] = [];
if (pathsByTrack.system) playbackPaths.push(pathsByTrack.system);
if (pathsByTrack.mic) playbackPaths.push(pathsByTrack.mic);
if (!hasDedicatedTracks && pathsByTrack.mixed) playbackPaths.push(pathsByTrack.mixed);
const includeEmbeddedInExport = !pathsByTrack.system && !pathsByTrack.mixed;
const resolvedRegions = (input.audioRegions ?? []).slice().sort((a, b) => a.startMs - b.startMs);
const tracks: ResolvedAudioTrack[] = resolvedRegions.map((region) => ({
id: `user:${region.id}`,
kind: "user",
sourceRef: {
path: region.audioPath,
startDelayMs: 0,
},
gain: clampGain(region.volume * (region.normalize ? SOURCE_AUDIO_NORMALIZE_GAIN : 1), 1),
timelineBinding: {
startMs: Math.max(0, region.startMs),
endMs: Math.max(0, region.endMs),
},
}));
for (const audioPath of playbackPaths) {
const trackId = getSourceTrackIdFromPath(audioPath);
tracks.push({
id: `${trackId}:${audioPath}`,
kind: trackId,
sourceRef: {
path: audioPath,
startDelayMs: 0,
},
gain: clampGain(input.sourceTrackGainById?.[trackId] ?? 1, 2),
timelineBinding: {
startMs: 0,
endMs: Number.POSITIVE_INFINITY,
},
});
}
if (hasEmbeddedSourceAudio && input.videoResource) {
tracks.push({
id: `embedded:${input.videoResource}`,
kind: "embedded",
sourceRef: {
path: input.videoResource,
startDelayMs: 0,
},
gain: clampGain(input.embeddedGain ?? input.sourceTrackGainById?.mixed ?? 1, 2),
timelineBinding: {
startMs: 0,
endMs: Number.POSITIVE_INFINITY,
},
});
}
return {
hasEmbeddedSourceAudio,
pathsByTrack,
playbackPaths,
muteEmbeddedPreview: hasDedicatedTracks && !includeEmbeddedInExport,
includeEmbeddedInExport,
tracks,
masterGain: clampGain(input.masterGain ?? 1, 1),
};
}
+35 -11
View File
@@ -3,11 +3,13 @@ import type {
AudioRegion,
AutoCaptionSettings,
CaptionCue,
ClipRegion,
CropRegion,
CursorStyle,
CursorTelemetryPoint,
Padding,
SpeedRegion,
SourceAudioTrackSettings,
TrimRegion,
WebcamOverlaySettings,
ZoomMotionBlurTuning,
@@ -135,8 +137,10 @@ interface VideoExporterConfig extends ExportConfig {
zoomClassicMode?: boolean;
frame?: string | null;
audioRegions?: AudioRegion[];
clipRegions?: ClipRegion[];
sourceAudioFallbackPaths?: string[];
sourceAudioFallbackStartDelayMsByPath?: Record<string, number>;
sourceAudioTrackSettings?: SourceAudioTrackSettings;
previewWidth?: number;
previewHeight?: number;
onProgress?: (progress: ExportProgress) => void;
@@ -167,6 +171,16 @@ type NativeAudioPlan =
};
const FILTERGRAPH_FALLBACK_AUDIO_SAMPLE_RATE = 48_000;
function hasNonDefaultSourceTrackSettings(sourceAudioTrackSettings?: SourceAudioTrackSettings) {
if (!sourceAudioTrackSettings) {
return false;
}
return Object.values(sourceAudioTrackSettings).some(
(settings) =>
Math.abs((settings?.volume ?? 1) - 1) > 0.0005 || Boolean(settings?.normalize),
);
}
const MIN_NATIVE_STATIC_LAYOUT_SPEED = 0.25;
const MAX_NATIVE_STATIC_LAYOUT_SPEED = 30;
@@ -752,6 +766,8 @@ export class ModernVideoExporter {
this.config.audioRegions,
this.config.sourceAudioFallbackPaths,
this.config.sourceAudioFallbackStartDelayMsByPath,
this.config.sourceAudioTrackSettings,
this.config.clipRegions,
),
"audio processing",
"audio",
@@ -1181,7 +1197,9 @@ export class ModernVideoExporter {
speedRegions.length > 0 ||
audioRegions.length > 0 ||
sourceAudioFallbackPaths.length > 1 ||
hasTimedSourceAudioFallback
hasTimedSourceAudioFallback ||
hasNonDefaultSourceTrackSettings(this.config.sourceAudioTrackSettings) ||
(this.config.clipRegions ?? []).some((clip) => Boolean(clip.muted))
) {
const sourceDurationMs = Math.max(
0,
@@ -1201,16 +1219,20 @@ export class ModernVideoExporter {
typeof primaryAudioSourceSampleRate === "number" &&
Number.isFinite(primaryAudioSourceSampleRate) &&
primaryAudioSourceSampleRate > 0;
const strategy = canUsePrimaryAudioFiltergraph
? classifyEditedTrackStrategy({
primaryAudioSourcePath,
sourceDurationMs,
trimRegions,
speedRegions,
audioRegions,
sourceAudioFallbackPaths,
})
: "offline-render-fallback";
const requiresRenderedEditedTrack =
hasNonDefaultSourceTrackSettings(this.config.sourceAudioTrackSettings) ||
(this.config.clipRegions ?? []).some((clip) => Boolean(clip.muted));
const strategy =
canUsePrimaryAudioFiltergraph && !requiresRenderedEditedTrack
? classifyEditedTrackStrategy({
primaryAudioSourcePath,
sourceDurationMs,
trimRegions,
speedRegions,
audioRegions,
sourceAudioFallbackPaths,
})
: "offline-render-fallback";
if (strategy === "filtergraph-fast-path") {
const audioSourcePath = primaryAudioSourcePath;
@@ -1805,6 +1827,8 @@ export class ModernVideoExporter {
this.config.audioRegions,
this.config.sourceAudioFallbackPaths,
this.config.sourceAudioFallbackStartDelayMsByPath,
this.config.sourceAudioTrackSettings,
this.config.clipRegions,
),
description,
"audio",
@@ -0,0 +1,41 @@
import { describe, expect, it } from "vitest";
import { resolveSourceTrackRoutingPolicy } from "./sourceTrackRoutingPolicy";
describe("resolveSourceTrackRoutingPolicy", () => {
it("prioritizes system+mic sidecars and mutes embedded preview", () => {
const policy = resolveSourceTrackRoutingPolicy("/tmp/recording.mp4", [
"/tmp/recording.mp4",
"/tmp/recording.system.wav",
"/tmp/recording.mic.wav",
"/tmp/recording.mixed.wav",
]);
expect(policy.playbackPaths).toEqual([
"/tmp/recording.system.wav",
"/tmp/recording.mic.wav",
]);
expect(policy.muteEmbeddedPreview).toBe(true);
expect(policy.includeEmbeddedInExport).toBe(false);
});
it("falls back to mixed when dedicated tracks are absent", () => {
const policy = resolveSourceTrackRoutingPolicy("/tmp/recording.mp4", [
"/tmp/recording.mixed.wav",
]);
expect(policy.playbackPaths).toEqual(["/tmp/recording.mixed.wav"]);
expect(policy.muteEmbeddedPreview).toBe(false);
expect(policy.includeEmbeddedInExport).toBe(false);
});
it("keeps embedded audio when only mic sidecar is present", () => {
const policy = resolveSourceTrackRoutingPolicy("/tmp/recording.mp4", [
"/tmp/recording.mp4",
"/tmp/recording.mic.wav",
]);
expect(policy.playbackPaths).toEqual(["/tmp/recording.mic.wav"]);
expect(policy.muteEmbeddedPreview).toBe(false);
expect(policy.includeEmbeddedInExport).toBe(true);
});
});
@@ -0,0 +1,30 @@
import {
buildResolvedAudioPlan,
type SourceTrackId,
} from "./audioRoutingEngine";
export interface SourceTrackRoutingPolicy {
hasEmbeddedSourceAudio: boolean;
pathsByTrack: Partial<Record<SourceTrackId, string>>;
playbackPaths: string[];
muteEmbeddedPreview: boolean;
includeEmbeddedInExport: boolean;
}
export function resolveSourceTrackRoutingPolicy(
videoResource: string | null | undefined,
sourceAudioFallbackPaths: string[] | null | undefined,
): SourceTrackRoutingPolicy {
const plan = buildResolvedAudioPlan({
videoResource,
sourceAudioFallbackPaths,
});
return {
hasEmbeddedSourceAudio: plan.hasEmbeddedSourceAudio,
pathsByTrack: plan.pathsByTrack,
playbackPaths: plan.playbackPaths,
muteEmbeddedPreview: plan.muteEmbeddedPreview,
includeEmbeddedInExport: plan.includeEmbeddedInExport,
};
}
+22 -1
View File
@@ -3,11 +3,13 @@ import type {
AudioRegion,
AutoCaptionSettings,
CaptionCue,
ClipRegion,
CropRegion,
CursorStyle,
CursorTelemetryPoint,
Padding,
SpeedRegion,
SourceAudioTrackSettings,
TrimRegion,
WebcamOverlaySettings,
ZoomMotionBlurTuning,
@@ -90,8 +92,10 @@ interface VideoExporterConfig extends ExportConfig {
zoomSmoothness?: number;
frame?: string | null;
audioRegions?: AudioRegion[];
clipRegions?: ClipRegion[];
sourceAudioFallbackPaths?: string[];
sourceAudioFallbackStartDelayMsByPath?: Record<string, number>;
sourceAudioTrackSettings?: SourceAudioTrackSettings;
previewWidth?: number;
previewHeight?: number;
onProgress?: (progress: ExportProgress) => void;
@@ -121,6 +125,16 @@ type NativeAudioPlan =
const FILTERGRAPH_FALLBACK_AUDIO_SAMPLE_RATE = 48_000;
function hasNonDefaultSourceTrackSettings(sourceAudioTrackSettings?: SourceAudioTrackSettings) {
if (!sourceAudioTrackSettings) {
return false;
}
return Object.values(sourceAudioTrackSettings).some(
(settings) =>
Math.abs((settings?.volume ?? 1) - 1) > 0.0005 || Boolean(settings?.normalize),
);
}
export class VideoExporter {
private config: VideoExporterConfig;
private streamingDecoder: StreamingVideoDecoder | null = null;
@@ -398,6 +412,7 @@ export class VideoExporter {
this.config.audioRegions,
this.config.sourceAudioFallbackPaths,
this.config.sourceAudioFallbackStartDelayMsByPath,
this.config.sourceAudioTrackSettings,
),
"audio processing",
"audio",
@@ -560,7 +575,9 @@ export class VideoExporter {
speedRegions.length > 0 ||
audioRegions.length > 0 ||
sourceAudioFallbackPaths.length > 1 ||
hasTimedSourceAudioFallback
hasTimedSourceAudioFallback ||
hasNonDefaultSourceTrackSettings(this.config.sourceAudioTrackSettings) ||
(this.config.clipRegions ?? []).some((clip) => Boolean(clip.muted))
) {
const sourceDurationMs = Math.max(
0,
@@ -847,6 +864,8 @@ export class VideoExporter {
this.config.audioRegions,
this.config.sourceAudioFallbackPaths,
this.config.sourceAudioFallbackStartDelayMsByPath,
this.config.sourceAudioTrackSettings,
this.config.clipRegions,
),
"native edited audio rendering",
"audio",
@@ -943,6 +962,8 @@ export class VideoExporter {
this.config.audioRegions,
this.config.sourceAudioFallbackPaths,
this.config.sourceAudioFallbackStartDelayMsByPath,
this.config.sourceAudioTrackSettings,
this.config.clipRegions,
),
"ffmpeg edited audio rendering",
"audio",