diff --git a/electron/ipc/project/manager.test.ts b/electron/ipc/project/manager.test.ts index 12059f7a..3fa810da 100644 --- a/electron/ipc/project/manager.test.ts +++ b/electron/ipc/project/manager.test.ts @@ -172,4 +172,34 @@ describe("local media path policy", () => { expect(result.path).toBe(projectPath); expect(result.project).toMatchObject({ videoPath }); }); + + it("approves editor audioRegions audioPath entries when loading a project", async () => { + const downloadsPath = path.join(tempRoot, "Downloads"); + const videoPath = path.join(tempPath, "recording.mp4"); + const audioPath = path.join(downloadsPath, "music.ogg"); + const projectPath = path.join(tempPath, "recording.recordly"); + await fs.mkdir(downloadsPath, { recursive: true }); + await fs.writeFile(videoPath, "test-video"); + await fs.writeFile(audioPath, "test-audio"); + await fs.writeFile( + projectPath, + JSON.stringify({ + version: 1, + videoPath, + editor: { + audioRegions: [ + { id: "a1", startMs: 0, endMs: 1000, audioPath, volume: 1 }, + ], + }, + }), + "utf-8", + ); + + const { loadProjectFromPath, resolveApprovedLocalMediaPath } = await import("./manager"); + const resolvedAudioPath = await fs.realpath(audioPath); + + const result = await loadProjectFromPath(projectPath); + expect(result.success).toBe(true); + await expect(resolveApprovedLocalMediaPath(audioPath)).resolves.toBe(resolvedAudioPath); + }); }); diff --git a/electron/ipc/project/manager.ts b/electron/ipc/project/manager.ts index 404654b1..2f6e4fc2 100644 --- a/electron/ipc/project/manager.ts +++ b/electron/ipc/project/manager.ts @@ -428,6 +428,7 @@ export async function loadProjectFromPath(projectPath: string) { const projectObj = project as Record; const editorObj = projectObj?.editor as Record | undefined; const audioTracks = editorObj?.audioTracks as { sourcePath?: unknown }[] | undefined; + const audioRegions = editorObj?.audioRegions as { audioPath?: unknown }[] | undefined; const approvedProjectPaths: Array = [ mediaSources.videoPath, mediaSources.webcamPath, @@ -439,6 +440,13 @@ export async function loadProjectFromPath(projectPath: string) { } } } + if (Array.isArray(audioRegions)) { + for (const region of audioRegions) { + if (typeof region?.audioPath === "string") { + approvedProjectPaths.push(region.audioPath); + } + } + } await replaceApprovedSessionLocalReadPaths(approvedProjectPaths); await rememberRecentProject(normalizedPath); diff --git a/electron/ipc/recording/audioFilters.ts b/electron/ipc/recording/audioFilters.ts index 5da2a29e..b465f5f2 100644 --- a/electron/ipc/recording/audioFilters.ts +++ b/electron/ipc/recording/audioFilters.ts @@ -32,7 +32,7 @@ export function getBrowserMicSidecarFilters(profile?: string | null) { return BROWSER_MIC_SIDECAR_FILTERS; } -export const RECORDING_AUDIO_SIDECAR_DEBUG_ENV = "RECORDLY_KEEP_RECORDING_AUDIO_SIDECARS"; +export const RECORDING_AUDIO_SIDECAR_DEBUG_ENV = "RECORDLY_KEEP_RECORDING_AUDIO_SIDECARS"; // not used yet, because we need to have seperate audio files for system and mic for each recording export function shouldKeepRecordingAudioSidecars(env: NodeJS.ProcessEnv = process.env) { const value = env[RECORDING_AUDIO_SIDECAR_DEBUG_ENV]?.trim().toLowerCase(); diff --git a/electron/ipc/recording/diagnostics.ts b/electron/ipc/recording/diagnostics.ts index b8fb6860..f097cea9 100644 --- a/electron/ipc/recording/diagnostics.ts +++ b/electron/ipc/recording/diagnostics.ts @@ -499,20 +499,21 @@ export async function getCompanionAudioFallbackInfo(videoPath: string) { let paths: string[]; if (await hasEmbeddedAudioStream(videoPath)) { - const microphoneCompanionPaths = Array.from( + const companionPaths = Array.from( new Set( companionCandidates.flatMap((candidate) => candidate.usablePaths.filter( - (companionPath) => companionPath === candidate.micPath, + (companionPath) => + companionPath === candidate.micPath || companionPath === candidate.systemPath, ), ), ), ); - if (microphoneCompanionPaths.length === 0) { + if (companionPaths.length === 0) { return { paths: [], startDelayMsByPath: {} }; } - paths = [videoPath, ...microphoneCompanionPaths]; + paths = [videoPath, ...companionPaths]; } else { paths = Array.from( new Set(companionCandidates.flatMap((candidate) => candidate.usablePaths)), diff --git a/electron/ipc/recording/mac.ts b/electron/ipc/recording/mac.ts index 769d22ca..1227b0af 100644 --- a/electron/ipc/recording/mac.ts +++ b/electron/ipc/recording/mac.ts @@ -277,12 +277,6 @@ export async function muxNativeMacRecordingWithAudio( await moveFileWithOverwrite(mixedOutputPath, videoPath); console.log("[mux] Successfully muxed audio into video:", videoPath); - - for (const audioPath of [systemAudioPath, microphonePath]) { - if (audioPath) { - await fs.rm(audioPath, { force: true }).catch(() => undefined); - } - } } export function attachNativeCaptureLifecycle(process: ChildProcessWithoutNullStreams) { diff --git a/electron/ipc/recording/windows.ts b/electron/ipc/recording/windows.ts index 6fc6c035..5b41fc4d 100644 --- a/electron/ipc/recording/windows.ts +++ b/electron/ipc/recording/windows.ts @@ -24,7 +24,6 @@ import { import type { AudioSyncAdjustment } from "../types"; import { moveFileWithOverwrite } from "../utils"; import { - RECORDING_AUDIO_SIDECAR_DEBUG_ENV, shouldKeepRecordingAudioSidecars, WINDOWS_NATIVE_MIC_PRE_FILTERS, } from "./audioFilters"; @@ -474,30 +473,6 @@ export async function muxNativeWindowsVideoWithAudio( throw error; } - if (keepAudioSidecars) { - console.log( - `[mux-win] Keeping native audio sidecars because ${RECORDING_AUDIO_SIDECAR_DEBUG_ENV} is enabled`, - ); - return { - muxed: true, - videoDurationSeconds: videoDuration, - muxTimeoutMs, - audioInputs, - audio, - outputPath: videoPath, - keptAudioSidecars: true, - }; - } - - for (const audioPath of [systemAudioPath, micAudioPath]) { - if (audioPath) { - await Promise.all([ - fs.rm(audioPath, { force: true }).catch(() => undefined), - fs.rm(`${audioPath}.json`, { force: true }).catch(() => undefined), - ]); - } - } - return { muxed: true, videoDurationSeconds: videoDuration, @@ -505,6 +480,6 @@ export async function muxNativeWindowsVideoWithAudio( audioInputs, audio, outputPath: videoPath, - keptAudioSidecars: false, + keptAudioSidecars: true, }; } diff --git a/src/components/video-editor/AnnotationSettingsPanel.tsx b/src/components/video-editor/AnnotationSettingsPanel.tsx index a0370a5d..db90b746 100644 --- a/src/components/video-editor/AnnotationSettingsPanel.tsx +++ b/src/components/video-editor/AnnotationSettingsPanel.tsx @@ -139,8 +139,9 @@ export function AnnotationSettingsPanel({ }; return ( -
-
+
+
+
{t("annotations.settings")} @@ -772,16 +773,6 @@ export function AnnotationSettingsPanel({ - -
@@ -795,6 +786,18 @@ export function AnnotationSettingsPanel({
  • {t("annotations.tipCycleBackward")}
    • +
    +
    +
    +
    ); diff --git a/src/components/video-editor/SettingsPanel.tsx b/src/components/video-editor/SettingsPanel.tsx index 71c9ce53..71e54109 100644 --- a/src/components/video-editor/SettingsPanel.tsx +++ b/src/components/video-editor/SettingsPanel.tsx @@ -469,12 +469,21 @@ interface SettingsPanelProps { selectedClipId?: string | null; selectedClipSpeed?: number | null; selectedClipMuted?: boolean | null; + selectedClipShowSourceAudio?: boolean | null; + hasClipSourceAudio?: boolean; onClipSpeedChange?: (speed: number) => void; onClipMutedChange?: (muted: boolean) => void; + onClipShowSourceAudioChange?: (show: boolean) => void; + sourceAudioTrackMeta?: Array<{ id: string; label: string }>; + sourceAudioTrackSettings?: Record; + onSourceAudioTrackVolumeChange?: (id: string, volume: number) => void; + onSourceAudioTrackNormalizeChange?: (id: string, normalize: boolean) => void; onClipDelete?: (id: string) => void; selectedAudioId?: string | null; selectedAudioVolume?: number | null; + selectedAudioNormalize?: boolean | null; onAudioVolumeChange?: (volume: number) => void; + onAudioNormalizeChange?: (normalize: boolean) => void; onAudioDelete?: (id: string) => void; shadowIntensity?: number; onShadowChange?: (intensity: number) => void; @@ -862,12 +871,21 @@ export function SettingsPanel({ selectedClipId, selectedClipSpeed, selectedClipMuted, + selectedClipShowSourceAudio = false, + hasClipSourceAudio = false, onClipSpeedChange, onClipMutedChange, + onClipShowSourceAudioChange, + sourceAudioTrackMeta = [], + sourceAudioTrackSettings = {}, + onSourceAudioTrackVolumeChange, + onSourceAudioTrackNormalizeChange, onClipDelete, selectedAudioId, selectedAudioVolume, + selectedAudioNormalize, onAudioVolumeChange, + onAudioNormalizeChange, onAudioDelete, shadowIntensity = 0.67, onShadowChange, @@ -3035,6 +3053,45 @@ export function SettingsPanel({ ); + const audioSectionContent = ( +
    +
    + {tSettings("audio.volumeTitle", "Audio")} + +
    + onAudioVolumeChange?.(v)} + formatValue={(v) => `${Math.round(v * 100)}%`} + parseInput={(text) => parseFloat(text.replace(/%$/, "")) / 100} + /> +
    + + {tSettings("audio.normalize", "Normalize")} + + onAudioNormalizeChange?.(v)} + className="data-[state=checked]:bg-[#2563EB] scale-75" + /> +
    +
    + ); + const clipSectionContent = (
    @@ -3045,16 +3102,7 @@ export function SettingsPanel({ )}
    -
    - - {tSettings("clip.muteAudio", "Mute Audio")} - - onClipMutedChange?.(v)} - className="data-[state=checked]:bg-[#06b6d4] scale-75" - /> -
    +
    {tSettings("speed.label", "Speed")}
    @@ -3095,19 +3143,100 @@ export function SettingsPanel({ ); })}
    - {selectedClipId && ( - - )} + +
    + {tSettings("audio.title", "Audio")} + +
    +
    + + {tSettings("clip.mute", "Mute")} + +

    + {selectedClipMuted + ? tSettings("clip.mutedState", "Audio is muted") + : tSettings("clip.unmutedState", "Audio is playing")} +

    +
    + onClipMutedChange?.(v)} + className="data-[state=checked]:bg-[#06b6d4] scale-75" + /> +
    + {hasClipSourceAudio && ( +
    + + {tSettings("clip.separateClipFromAudio", "Separate clip from audio")} + + onClipShowSourceAudioChange?.(v)} + className="data-[state=checked]:bg-[#06b6d4] scale-75" + /> +
    + )} +
    + + {selectedClipId && + hasClipSourceAudio && + sourceAudioTrackMeta.length > 0 && ( +
    + {sourceAudioTrackMeta.map((track) => { + const settings = sourceAudioTrackSettings[track.id] ?? { + volume: 1, + normalize: false, + }; + return ( +
    +
    + + {track.label} + + +
    +
    + + {tSettings("audio.normalize", "Normalize")} + + + onSourceAudioTrackNormalizeChange?.(track.id, v) + } + className="data-[state=checked]:bg-[#06b6d4] scale-75" + /> +
    + onSourceAudioTrackVolumeChange?.(track.id, v)} + formatValue={(v) => `${Math.round(v * 100)}%`} + parseInput={(text) => + parseFloat(text.replace(/%$/, "")) / 100 + } + /> +
    + ); + })} +
    + )} ); @@ -3120,6 +3249,8 @@ export function SettingsPanel({ return zoomItemSectionContent; case "clip": return clipSectionContent; + case "audio": + return audioSectionContent; case "frame": return sceneSectionContent; case "crop": @@ -3561,41 +3692,68 @@ export function SettingsPanel({
    { + if (activeEffectSection === "clip" && selectedClipId) return false; + if (activeEffectSection === "zoom" && selectedZoomId) return false; + if (activeEffectSection === "audio" && selectedAudioId) return false; + if (selectedAnnotationId) return false; // Annotation editor handles its own but let's see + return true; + })() && "hidden", )} > - {selectedAudioId && ( -
    -
    - - {tSettings("audio.volumeTitle", "Audio Volume")} - - - {Math.round((selectedAudioVolume ?? 1) * 100)}% - -
    - onAudioVolumeChange?.(v)} - formatValue={(v) => `${Math.round(v * 100)}%`} - parseInput={(text) => parseFloat(text.replace(/%$/, "")) / 100} - /> - -
    + {activeEffectSection === "clip" && selectedClipId && ( + + )} + {activeEffectSection === "zoom" && selectedZoomId && ( + + )} + {activeEffectSection === "audio" && selectedAudioId && ( + + )} + {selectedAnnotationId && ( + )}
    diff --git a/src/components/video-editor/VideoEditor.tsx b/src/components/video-editor/VideoEditor.tsx index fa5fc5ac..71a29d75 100644 --- a/src/components/video-editor/VideoEditor.tsx +++ b/src/components/video-editor/VideoEditor.tsx @@ -80,14 +80,6 @@ import { canUseInMemoryExportSaveFallback, describeBlockedInMemoryExportSave, } from "@/lib/exporter/exportSavePolicy"; -import { resolveMediaElementSource } from "@/lib/exporter/localMediaSource"; -import { resolveSourceAudioFallbackPaths } from "@/lib/exporter/sourceAudioFallback"; -import { - clampMediaTimeToDuration, - enablePitchPreservingPlayback, - estimateCompanionAudioStartDelaySeconds, - getMediaSyncPlaybackRate, -} from "@/lib/mediaTiming"; import { matchesShortcut } from "@/lib/shortcuts"; import { cn } from "@/lib/utils"; import { @@ -147,6 +139,7 @@ import { validateProjectData, } from "./projectPersistence"; import { SettingsPanel } from "./SettingsPanel"; +import { useVideoEditorAudio } from "./audio/useVideoEditorAudio"; import { APP_HEADER_ICON_BUTTON_CLASS, DiscordLinkButton, @@ -156,6 +149,7 @@ import { } from "./TutorialHelp"; import TimelineEditor, { type TimelineEditorHandle } from "./timeline/TimelineEditor"; import { normalizeCursorTelemetry } from "./timeline/zoomSuggestionUtils"; +import type { SourceAudioTrackSettings } from "@/components/video-editor/audio/audioTypes"; import { type AnnotationRegion, type AudioRegion, @@ -347,12 +341,6 @@ async function writeSmokeExportReport( const SMOKE_EXPORT_READY_TIMEOUT_MS = 30_000; const DEFAULT_MP4_EXPORT_FRAME_RATE: ExportMp4FrameRate = 30; -const SOURCE_AUDIO_FALLBACK_TOAST_ID = "source-audio-fallback-error"; -const SOURCE_AUDIO_PREVIEW_PLAYING_SEEK_DRIFT_SECONDS = 0.18; -const SOURCE_AUDIO_PREVIEW_PAUSED_SEEK_DRIFT_SECONDS = 0.01; -const SOURCE_AUDIO_PREVIEW_RATE_TOLERANCE_SECONDS = 0.08; -const SOURCE_AUDIO_PREVIEW_RATE_CORRECTION_WINDOW_SECONDS = 8; -const SOURCE_AUDIO_PREVIEW_MAX_RATE_ADJUSTMENT = 0.015; const PROJECT_AUTOSAVE_DELAY_MS = 1000; const EXPORT_ERROR_TOAST_DURATION_MS = 20000; @@ -677,6 +665,13 @@ export default function VideoEditor() { const [selectedAnnotationId, setSelectedAnnotationId] = useState(null); const [audioRegions, setAudioRegions] = useState([]); const [selectedAudioId, setSelectedAudioId] = useState(null); + const [sourceAudioTrackSettingsByClip, setSourceAudioTrackSettingsByClip] = useState< + Record + >({}); + const [defaultSourceAudioTrackSettings, setDefaultSourceAudioTrackSettings] = useState< + SourceAudioTrackSettings + >({}); + const [hasClipSourceAudio, setHasClipSourceAudio] = useState(false); const [autoCaptions, setAutoCaptions] = useState([]); const [autoCaptionSettings, setAutoCaptionSettings] = useState( DEFAULT_AUTO_CAPTION_SETTINGS, @@ -700,9 +695,6 @@ export default function VideoEditor() { const [exportError, setExportError] = useState(null); const [showExportDropdown, setShowExportDropdown] = useState(false); const [previewVolume, setPreviewVolume] = useState(1); - const [sourceAudioFallbackPaths, setSourceAudioFallbackPaths] = useState([]); - const [sourceAudioFallbackStartDelayMsByPath, setSourceAudioFallbackStartDelayMsByPath] = - useState>({}); const applySessionPresentation = useCallback( ( session: @@ -1767,6 +1759,8 @@ export default function VideoEditor() { gifFrameRate: GifFrameRate; gifLoop: boolean; gifSizePreset: GifSizePreset; + sourceAudioTrackSettingsByClip: Record; + defaultSourceAudioTrackSettings: SourceAudioTrackSettings; }>, ) => { return editor; @@ -1778,63 +1772,6 @@ export default function VideoEditor() { () => videoSourcePath ?? (videoPath ? fromFileUrl(videoPath) : null), [videoPath, videoSourcePath], ); - const { hasEmbeddedSourceAudio, externalAudioPaths: previewSourceAudioFallbackPaths } = useMemo( - () => resolveSourceAudioFallbackPaths(currentSourcePath, sourceAudioFallbackPaths), - [currentSourcePath, sourceAudioFallbackPaths], - ); - const shouldMutePreviewVideo = - !hasEmbeddedSourceAudio && previewSourceAudioFallbackPaths.length > 0; - - useEffect(() => { - let cancelled = false; - setSourceAudioFallbackPaths([]); - setSourceAudioFallbackStartDelayMsByPath({}); - - if (!currentSourcePath) { - return () => { - cancelled = true; - }; - } - - void (async () => { - try { - const result = - await window.electronAPI.getVideoAudioFallbackPaths(currentSourcePath); - if (cancelled) { - return; - } - if (!result.success) { - setSourceAudioFallbackPaths([]); - setSourceAudioFallbackStartDelayMsByPath({}); - toast.warning( - result.error - ? `Could not load companion audio sources: ${summarizeErrorMessage(result.error)}` - : "Could not load companion audio sources. Playback and export may miss microphone audio.", - { id: SOURCE_AUDIO_FALLBACK_TOAST_ID, duration: 10000 }, - ); - return; - } - - toast.dismiss(SOURCE_AUDIO_FALLBACK_TOAST_ID); - setSourceAudioFallbackPaths(result.paths ?? []); - setSourceAudioFallbackStartDelayMsByPath(result.startDelayMsByPath ?? {}); - } catch (error) { - if (!cancelled) { - setSourceAudioFallbackPaths([]); - setSourceAudioFallbackStartDelayMsByPath({}); - toast.warning( - `Could not load companion audio sources: ${summarizeErrorMessage(String(error))}`, - { id: SOURCE_AUDIO_FALLBACK_TOAST_ID, duration: 10000 }, - ); - } - } - })(); - - return () => { - cancelled = true; - }; - }, [currentSourcePath]); - const projectDisplayName = useMemo(() => { const fileName = currentProjectPath?.split(/[\\/]/).pop() ?? @@ -1925,6 +1862,8 @@ export default function VideoEditor() { gifFrameRate, gifLoop, gifSizePreset, + sourceAudioTrackSettingsByClip, + defaultSourceAudioTrackSettings, }), [ buildPersistedEditorState, @@ -1985,6 +1924,8 @@ export default function VideoEditor() { gifLoop, gifSizePreset, frame, + sourceAudioTrackSettingsByClip, + defaultSourceAudioTrackSettings, ], ); @@ -2170,6 +2111,10 @@ export default function VideoEditor() { setSpeedRegions(normalizedEditor.speedRegions); setAnnotationRegions(normalizedEditor.annotationRegions); setAudioRegions(normalizedEditor.audioRegions); + setSourceAudioTrackSettingsByClip(normalizedEditor.sourceAudioTrackSettingsByClip ?? {}); + setDefaultSourceAudioTrackSettings( + normalizedEditor.defaultSourceAudioTrackSettings ?? {}, + ); setAutoCaptions(normalizedEditor.autoCaptions); setAutoCaptionSettings(normalizedEditor.autoCaptionSettings); setAspectRatio(normalizedEditor.aspectRatio); @@ -3380,6 +3325,29 @@ export default function VideoEditor() { } return result; }, [clipRegions, speedRegions]); + const audio = useVideoEditorAudio({ + currentSourcePath, + selectedClipId, + clipRegions, + audioRegions, + effectiveSpeedRegions, + sourceAudioTrackSettingsByClip, + setSourceAudioTrackSettingsByClip, + defaultSourceAudioTrackSettings, + setDefaultSourceAudioTrackSettings, + currentTime, + timelineTime: timelinePlayheadTime, + duration, + isPlaying, + previewVolume, + summarizeErrorMessage, + onSourceFallbackLoadError: (error) => { + toast.warning( + `Could not load companion audio source: ${summarizeErrorMessage(getErrorMessage(error))}`, + { duration: 10000 }, + ); + }, + }); function togglePlayPause() { const playback = videoPlaybackRef.current; @@ -3761,6 +3729,18 @@ export default function VideoEditor() { [selectedClipId], ); + const handleClipShowSourceAudioChange = useCallback( + (showSourceAudio: boolean) => { + if (!selectedClipId) return; + setClipRegions((prev) => + prev.map((clip) => + clip.id === selectedClipId ? { ...clip, showSourceAudio } : clip, + ), + ); + }, + [selectedClipId], + ); + const handleClipDelete = useCallback( (id: string) => { const deletedClip = clipRegions.find((clip) => clip.id === id); @@ -3792,23 +3772,26 @@ export default function VideoEditor() { if (id) { setSelectedZoomId(null); setSelectedAnnotationId(null); + setActiveEffectSection("audio"); } }, []); - const handleAudioAdded = useCallback((span: Span, audioPath: string, trackIndex?: number) => { - const id = `audio-${nextAudioIdRef.current++}`; - const newRegion: AudioRegion = { - id, - startMs: Math.round(span.start), - endMs: Math.round(span.end), - audioPath, - volume: 1, - trackIndex, - }; + const handleAudioAdded = useCallback((span: Span, audioPath: string, trackIndex?: number) => { + const id = `audio-${nextAudioIdRef.current++}`; + const newRegion: AudioRegion = { + id, + startMs: Math.round(span.start), + endMs: Math.round(span.end), + audioPath, + volume: 1, + normalize: false, + trackIndex, + }; setAudioRegions((prev) => [...prev, newRegion]); setSelectedAudioId(id); setSelectedZoomId(null); setSelectedAnnotationId(null); + setActiveEffectSection("audio"); }, []); const handleAudioSpanChange = useCallback((id: string, span: Span, trackIndex?: number) => { @@ -3853,15 +3836,29 @@ export default function VideoEditor() { [selectedAudioId], ); - const handleAudioDelete = useCallback( - (id: string) => { + const handleAudioDelete = useCallback( + (id: string) => { setAudioRegions((prev) => prev.filter((region) => region.id !== id)); if (selectedAudioId === id) { setSelectedAudioId(null); } }, - [selectedAudioId], - ); + [selectedAudioId], + ); + + const handleAudioNormalizeChange = useCallback( + (normalize: boolean) => { + if (!selectedAudioId) { + return; + } + setAudioRegions((prev) => + prev.map((region) => + region.id === selectedAudioId ? { ...region, normalize } : region, + ), + ); + }, + [selectedAudioId], + ); const handleAnnotationAdded = useCallback((span: Span, trackIndex = 0) => { const id = `annotation-${nextAnnotationIdRef.current++}`; @@ -4096,290 +4093,6 @@ export default function VideoEditor() { } }, [selectedAudioId, audioRegions]); - // Audio playback sync: manage Audio elements that play in sync with video - const audioElementsRef = useRef>(new Map()); - const audioElementRevokersRef = useRef void>>(new Map()); - const audioElementResourcesRef = useRef>(new Map()); - const sourceAudioElementsRef = useRef>(new Map()); - const sourceAudioElementRevokersRef = useRef void>>(new Map()); - const sourceAudioElementResourcesRef = useRef>(new Map()); - const lastSourceAudioSyncTimeRef = useRef(null); - - useEffect(() => { - let cancelled = false; - const existing = audioElementsRef.current; - const currentIds = new Set(audioRegions.map((r) => r.id)); - - // Remove old audio elements - for (const [id, audio] of existing) { - if (!currentIds.has(id)) { - audio.pause(); - audio.src = ""; - audioElementRevokersRef.current.get(id)?.(); - audioElementRevokersRef.current.delete(id); - audioElementResourcesRef.current.delete(id); - existing.delete(id); - } - } - - // Create/update audio elements - for (const region of audioRegions) { - let audio = existing.get(region.id); - if (!audio) { - audio = new Audio(); - audio.preload = "auto"; - existing.set(region.id, audio); - } - - if (audioElementResourcesRef.current.get(region.id) !== region.audioPath) { - audio.pause(); - audio.src = ""; - audioElementRevokersRef.current.get(region.id)?.(); - audioElementRevokersRef.current.delete(region.id); - audioElementResourcesRef.current.set(region.id, region.audioPath); - - void (async () => { - const resolved = await resolveMediaElementSource(region.audioPath); - const latestAudio = existing.get(region.id); - - if ( - cancelled || - latestAudio !== audio || - audioElementResourcesRef.current.get(region.id) !== region.audioPath - ) { - resolved.revoke(); - return; - } - - audioElementRevokersRef.current.set(region.id, resolved.revoke); - latestAudio.src = resolved.src; - })(); - } - - audio.volume = Math.max(0, Math.min(1, region.volume * previewVolume)); - } - - return () => { - cancelled = true; - }; - }, [audioRegions, previewVolume]); - - useEffect(() => { - let cancelled = false; - const existing = sourceAudioElementsRef.current; - const currentIds = new Set(previewSourceAudioFallbackPaths); - - for (const [id, audio] of existing) { - if (!currentIds.has(id)) { - audio.pause(); - audio.src = ""; - sourceAudioElementRevokersRef.current.get(id)?.(); - sourceAudioElementRevokersRef.current.delete(id); - sourceAudioElementResourcesRef.current.delete(id); - existing.delete(id); - } - } - - for (const audioPath of previewSourceAudioFallbackPaths) { - let audio = existing.get(audioPath); - if (!audio) { - audio = new Audio(); - audio.preload = "auto"; - existing.set(audioPath, audio); - } - audio.dataset.sourceAudioPath = audioPath; - - if (sourceAudioElementResourcesRef.current.get(audioPath) !== audioPath) { - audio.pause(); - audio.src = ""; - sourceAudioElementRevokersRef.current.get(audioPath)?.(); - sourceAudioElementRevokersRef.current.delete(audioPath); - sourceAudioElementResourcesRef.current.set(audioPath, audioPath); - - void (async () => { - try { - const resolved = await resolveMediaElementSource(audioPath); - const latestAudio = existing.get(audioPath); - - if ( - cancelled || - latestAudio !== audio || - sourceAudioElementResourcesRef.current.get(audioPath) !== audioPath - ) { - resolved.revoke(); - return; - } - - sourceAudioElementRevokersRef.current.set(audioPath, resolved.revoke); - latestAudio.src = resolved.src; - } catch (error) { - if (cancelled) { - return; - } - - sourceAudioElementRevokersRef.current.get(audioPath)?.(); - sourceAudioElementRevokersRef.current.delete(audioPath); - sourceAudioElementResourcesRef.current.delete(audioPath); - const latestAudio = existing.get(audioPath); - if (latestAudio === audio) { - latestAudio.pause(); - latestAudio.src = ""; - } - toast.warning( - `Could not load companion audio source: ${summarizeErrorMessage(getErrorMessage(error))}`, - { id: SOURCE_AUDIO_FALLBACK_TOAST_ID, duration: 10000 }, - ); - } - })(); - } - - audio.volume = Math.max(0, Math.min(1, previewVolume)); - } - - if (previewSourceAudioFallbackPaths.length === 0) { - lastSourceAudioSyncTimeRef.current = null; - } - - return () => { - cancelled = true; - }; - }, [previewSourceAudioFallbackPaths, previewVolume]); - - useEffect(() => { - return () => { - for (const audio of audioElementsRef.current.values()) { - audio.pause(); - audio.src = ""; - } - for (const revoke of audioElementRevokersRef.current.values()) { - revoke(); - } - audioElementsRef.current.clear(); - audioElementRevokersRef.current.clear(); - audioElementResourcesRef.current.clear(); - for (const audio of sourceAudioElementsRef.current.values()) { - audio.pause(); - audio.src = ""; - } - for (const revoke of sourceAudioElementRevokersRef.current.values()) { - revoke(); - } - sourceAudioElementsRef.current.clear(); - sourceAudioElementRevokersRef.current.clear(); - sourceAudioElementResourcesRef.current.clear(); - lastSourceAudioSyncTimeRef.current = null; - }; - }, []); - - // Sync audio playback with video currentTime and isPlaying state - useEffect(() => { - const currentTimeMs = currentTime * 1000; - const activeSpeedRegion = effectiveSpeedRegions.find( - (region) => currentTimeMs >= region.startMs && currentTimeMs < region.endMs, - ); - const targetPlaybackRate = activeSpeedRegion ? activeSpeedRegion.speed : 1; - - for (const region of audioRegions) { - const audio = audioElementsRef.current.get(region.id); - if (!audio) continue; - - const isInRegion = currentTimeMs >= region.startMs && currentTimeMs < region.endMs; - - if (isPlaying && isInRegion) { - enablePitchPreservingPlayback(audio); - const audioOffset = (currentTimeMs - region.startMs) / 1000; - // Only seek if significantly out of sync (> 200ms) - if (Math.abs(audio.currentTime - audioOffset) > 0.2) { - audio.currentTime = audioOffset; - } - const syncedPlaybackRate = getMediaSyncPlaybackRate({ - basePlaybackRate: targetPlaybackRate, - currentTime: audio.currentTime, - targetTime: audioOffset, - }); - if (Math.abs(audio.playbackRate - syncedPlaybackRate) > 0.001) { - audio.playbackRate = syncedPlaybackRate; - } - if (audio.paused) { - audio.play().catch(() => undefined); - } - } else { - if (!audio.paused) { - audio.pause(); - } - } - } - }, [isPlaying, currentTime, audioRegions, effectiveSpeedRegions]); - - useEffect(() => { - if (previewSourceAudioFallbackPaths.length === 0) { - lastSourceAudioSyncTimeRef.current = null; - return; - } - - const activeSpeedRegion = effectiveSpeedRegions.find( - (region) => currentTime * 1000 >= region.startMs && currentTime * 1000 < region.endMs, - ); - const targetPlaybackRate = activeSpeedRegion ? activeSpeedRegion.speed : 1; - const previousTimelineTime = lastSourceAudioSyncTimeRef.current; - const timelineJumped = - previousTimelineTime === null || Math.abs(currentTime - previousTimelineTime) > 0.25; - const driftThreshold = isPlaying - ? SOURCE_AUDIO_PREVIEW_PLAYING_SEEK_DRIFT_SECONDS - : SOURCE_AUDIO_PREVIEW_PAUSED_SEEK_DRIFT_SECONDS; - - for (const audio of sourceAudioElementsRef.current.values()) { - enablePitchPreservingPlayback(audio); - const audioDuration = Number.isFinite(audio.duration) ? audio.duration : null; - const startDelaySeconds = estimateCompanionAudioStartDelaySeconds( - duration, - audioDuration, - sourceAudioFallbackStartDelayMsByPath[audio.dataset.sourceAudioPath ?? ""], - ); - const beforeAudioStart = currentTime + 0.001 < startDelaySeconds; - const targetTime = clampMediaTimeToDuration( - currentTime - startDelaySeconds, - audioDuration, - ); - - if (timelineJumped || Math.abs(audio.currentTime - targetTime) > driftThreshold) { - try { - audio.currentTime = targetTime; - } catch { - // no-op - } - } - - const syncedPlaybackRate = getMediaSyncPlaybackRate({ - basePlaybackRate: targetPlaybackRate, - currentTime: audio.currentTime, - targetTime, - toleranceSeconds: SOURCE_AUDIO_PREVIEW_RATE_TOLERANCE_SECONDS, - correctionWindowSeconds: SOURCE_AUDIO_PREVIEW_RATE_CORRECTION_WINDOW_SECONDS, - maxAdjustment: SOURCE_AUDIO_PREVIEW_MAX_RATE_ADJUSTMENT, - }); - if (Math.abs(audio.playbackRate - syncedPlaybackRate) > 0.001) { - audio.playbackRate = syncedPlaybackRate; - } - - const atEnd = audioDuration !== null && targetTime >= audioDuration; - if (isPlaying && !beforeAudioStart && !atEnd) { - audio.play().catch(() => undefined); - } else if (!audio.paused) { - audio.pause(); - } - } - - lastSourceAudioSyncTimeRef.current = currentTime; - }, [ - currentTime, - duration, - isPlaying, - previewSourceAudioFallbackPaths, - sourceAudioFallbackStartDelayMsByPath, - effectiveSpeedRegions, - ]); - const showExportSuccessToast = useCallback((filePath: string) => { toast.success(`Exported successfully to ${filePath}`, { action: { @@ -4639,6 +4352,10 @@ export default function VideoEditor() { encodingMode, useModernNativeStaticLayout: useExperimentalNativeExport, }); + const sourceAudioTrackSettingsForExport = + selectedClipId !== null + ? audio.selectedClipSourceAudioTrackSettings + : audio.activeSourceAudioTrackSettings; const exporterConfig = { videoUrl: videoPath, @@ -4704,8 +4421,11 @@ export default function VideoEditor() { cursorSway, frame, audioRegions, - sourceAudioFallbackPaths, - sourceAudioFallbackStartDelayMsByPath, + clipRegions, + sourceAudioFallbackPaths: audio.sourceAudioFallbackPaths, + sourceAudioFallbackStartDelayMsByPath: + audio.sourceAudioFallbackStartDelayMsByPath, + sourceAudioTrackSettings: sourceAudioTrackSettingsForExport, previewWidth, previewHeight, onProgress: (progress: ExportProgress) => { @@ -4952,8 +4672,10 @@ export default function VideoEditor() { cursorClickBounceDuration, cursorSway, audioRegions, - sourceAudioFallbackPaths, - sourceAudioFallbackStartDelayMsByPath, + audio.sourceAudioFallbackPaths, + audio.sourceAudioFallbackStartDelayMsByPath, + audio.activeSourceAudioTrackSettings, + audio.selectedClipSourceAudioTrackSettings, exportEncodingMode, exportBackendPreference, exportPipelineModel, @@ -4985,6 +4707,7 @@ export default function VideoEditor() { smokeExportConfig.shadowIntensity, effectiveSpeedRegions, frame, + selectedClipId, smokeExportConfig.encodingMode, smokeExportConfig.fps, smokeExportConfig.quality, @@ -6032,32 +5755,50 @@ export default function VideoEditor() { selectedClipId={selectedClipId} selectedClipSpeed={ selectedClipId - ? (clipRegions.find((c) => c.id === selectedClipId) - ?.speed ?? 1) + ? clipRegions.find((c) => c.id === selectedClipId)?.speed ?? 1 : null } selectedClipMuted={ selectedClipId - ? (clipRegions.find((c) => c.id === selectedClipId) - ?.muted ?? false) + ? clipRegions.find((c) => c.id === selectedClipId)?.muted ?? + false : null } - onClipSpeedChange={(speed) => - selectedClipId && handleClipSpeedChange(speed) - } - onClipMutedChange={(muted) => - selectedClipId && handleClipMutedChange(muted) + selectedClipShowSourceAudio={ + selectedClipId + ? clipRegions.find((c) => c.id === selectedClipId) + ?.showSourceAudio ?? false + : null } + onClipSpeedChange={handleClipSpeedChange} + onClipMutedChange={handleClipMutedChange} + onClipShowSourceAudioChange={handleClipShowSourceAudioChange} onClipDelete={handleClipDelete} - selectedAudioId={selectedAudioId} - selectedAudioVolume={ - selectedAudioId - ? (audioRegions.find((r) => r.id === selectedAudioId) - ?.volume ?? null) - : null + hasClipSourceAudio={hasClipSourceAudio} + sourceAudioTrackMeta={audio.sourceAudioTrackMeta} + sourceAudioTrackSettings={audio.selectedClipSourceAudioTrackSettings} + onSourceAudioTrackVolumeChange={ + audio.onSelectedClipSourceAudioTrackVolumeChange } - onAudioVolumeChange={handleAudioVolumeChange} - onAudioDelete={handleAudioDelete} + onSourceAudioTrackNormalizeChange={ + audio.onSelectedClipSourceAudioTrackNormalizeChange + } + selectedAudioId={selectedAudioId} + selectedAudioVolume={ + selectedAudioId + ? (audioRegions.find((r) => r.id === selectedAudioId) + ?.volume ?? null) + : null + } + selectedAudioNormalize={ + selectedAudioId + ? (audioRegions.find((r) => r.id === selectedAudioId) + ?.normalize ?? false) + : null + } + onAudioVolumeChange={handleAudioVolumeChange} + onAudioNormalizeChange={handleAudioNormalizeChange} + onAudioDelete={handleAudioDelete} shadowIntensity={shadowIntensity} onShadowChange={setShadowIntensity} backgroundBlur={backgroundBlur} @@ -6353,7 +6094,17 @@ export default function VideoEditor() { cursorClickBounceDuration } cursorSway={cursorSway} - volume={shouldMutePreviewVideo ? 0 : previewVolume} + volume={ + audio.shouldMutePreviewVideo || audio.isCurrentClipMuted + ? 0 + : Math.max( + 0, + Math.min( + 1, + previewVolume * audio.embeddedSourcePreviewGain, + ), + ) + } suspendRendering={shouldSuspendPreviewRendering} />
    @@ -6628,6 +6379,17 @@ export default function VideoEditor() { selectedAnnotationId={selectedAnnotationId} onSelectAnnotation={handleSelectAnnotation} aspectRatio={aspectRatio} + showSourceAudioTrack={clipRegions.some((c) => c.showSourceAudio)} + sourceAudioTrackSettings={audio.activeSourceAudioTrackSettings} + getSourceAudioTrackSettingsForClip={ + audio.getSourceAudioTrackSettingsForClip + } + onSourceAudioAvailabilityChange={(available) => { + setHasClipSourceAudio(available); + }} + onSourceAudioTracksMetaChange={(tracks) => { + audio.onSourceAudioTracksMetaChange(tracks); + }} />
    diff --git a/src/components/video-editor/audio/audioTypes.ts b/src/components/video-editor/audio/audioTypes.ts new file mode 100644 index 00000000..d20c224f --- /dev/null +++ b/src/components/video-editor/audio/audioTypes.ts @@ -0,0 +1,25 @@ +import type { AudioPeaksData } from "../timeline/core/timelineTypes"; + +export type SourceAudioTrackId = "mixed" | "system" | "mic" | (string & {}); + +export interface SourceAudioTrackSetting { + volume: number; + normalize: boolean; +} + +export type SourceAudioTrackSettings = Record; + +export interface SourceAudioTrackMetaItem { + id: SourceAudioTrackId; + label: string; +} + +export type SourceAudioTrackMeta = SourceAudioTrackMetaItem[]; + +export interface SourceAudioTrackWithPeaks extends SourceAudioTrackMetaItem { + peaks: AudioPeaksData; +} + +export const SOURCE_AUDIO_FALLBACK_TOAST_ID = "source-audio-fallback-error"; +export const SOURCE_AUDIO_NORMALIZE_GAIN = 1.35; + diff --git a/src/components/video-editor/audio/clipAudio.ts b/src/components/video-editor/audio/clipAudio.ts new file mode 100644 index 00000000..c6bf7409 --- /dev/null +++ b/src/components/video-editor/audio/clipAudio.ts @@ -0,0 +1,18 @@ +import { getClipSourceEndMs, sortClipRegions } from "../types"; +import type { ClipRegion } from "../types"; + +export function getActiveClipIdAtSourceTime( + sourceTimeSeconds: number, + clipRegions: ClipRegion[], +): string | null { + const sourceMs = Math.round(sourceTimeSeconds * 1000); + const activeClip = sortClipRegions(clipRegions).find( + (clip) => sourceMs >= clip.startMs && sourceMs < getClipSourceEndMs(clip), + ); + return activeClip?.id ?? null; +} + +export function isClipMutedById(clipId: string | null, clipRegions: ClipRegion[]): boolean { + if (!clipId) return false; + return clipRegions.find((clip) => clip.id === clipId)?.muted ?? false; +} diff --git a/src/components/video-editor/audio/useAudioPreviewSync.ts b/src/components/video-editor/audio/useAudioPreviewSync.ts new file mode 100644 index 00000000..78482460 --- /dev/null +++ b/src/components/video-editor/audio/useAudioPreviewSync.ts @@ -0,0 +1,459 @@ +import { useEffect, useMemo, useRef } from "react"; +import { buildResolvedAudioPlan } from "@/lib/exporter/audioRoutingEngine"; +import { resolveMediaElementSource } from "@/lib/exporter/localMediaSource"; +import { + clampMediaTimeToDuration, + enablePitchPreservingPlayback, + estimateCompanionAudioStartDelaySeconds, + getMediaSyncPlaybackRate, +} from "@/lib/mediaTiming"; +import type { AudioRegion, SpeedRegion } from "../types"; + +const SOURCE_AUDIO_PREVIEW_PLAYING_SEEK_DRIFT_SECONDS = 0.18; +const SOURCE_AUDIO_PREVIEW_PAUSED_SEEK_DRIFT_SECONDS = 0.01; + +interface UseAudioPreviewSyncParams { + audioRegions: AudioRegion[]; + previewVolume: number; + isPlaying: boolean; + currentTime: number; + timelineTime: number; + duration: number; + effectiveSpeedRegions: SpeedRegion[]; + previewSourceAudioFallbackPaths: string[]; + sourceAudioFallbackStartDelayMsByPath: Record; + isCurrentClipMuted: boolean; + getSourceTrackPreviewGain: (audioPath: string) => number; + onSourceFallbackLoadError: (error: unknown) => void; +} + +export function useAudioPreviewSync({ + audioRegions, + previewVolume, + isPlaying, + currentTime, + timelineTime, + duration, + effectiveSpeedRegions, + previewSourceAudioFallbackPaths, + sourceAudioFallbackStartDelayMsByPath, + isCurrentClipMuted, + getSourceTrackPreviewGain, + onSourceFallbackLoadError, +}: UseAudioPreviewSyncParams) { + const resolvedPlan = useMemo( + () => + buildResolvedAudioPlan({ + videoResource: null, + sourceAudioFallbackPaths: previewSourceAudioFallbackPaths, + audioRegions, + }), + [audioRegions, previewSourceAudioFallbackPaths], + ); + const resolvedUserTracks = useMemo( + () => resolvedPlan.tracks.filter((track) => track.kind === "user"), + [resolvedPlan], + ); + const resolvedSourceTracks = useMemo( + () => resolvedPlan.tracks.filter((track) => track.kind !== "user"), + [resolvedPlan], + ); + + const audioElementsRef = useRef>(new Map()); + const audioElementRevokersRef = useRef void>>(new Map()); + const audioElementResourcesRef = useRef>(new Map()); + const sourceAudioElementsRef = useRef>(new Map()); + const sourceAudioMediaNodesRef = useRef>(new Map()); + const sourceAudioGainNodesRef = useRef>(new Map()); + const sourceAudioElementRevokersRef = useRef void>>(new Map()); + const sourceAudioElementResourcesRef = useRef>(new Map()); + const sourceAudioContextRef = useRef(null); + const sourceAudioMasterGainRef = useRef(null); + const sourceAudioResumePromiseRef = useRef | null>(null); + const lastSourceAudioSyncTimeRef = useRef(null); + + const ensureSourceAudioContext = () => { + if (!sourceAudioContextRef.current) { + const context = new AudioContext({ latencyHint: "interactive" }); + const masterGain = context.createGain(); + masterGain.gain.value = 1; + masterGain.connect(context.destination); + sourceAudioContextRef.current = context; + sourceAudioMasterGainRef.current = masterGain; + } + return sourceAudioContextRef.current; + }; + + const ensureSourceAudioRunning = () => { + const context = ensureSourceAudioContext(); + if (context.state === "running") { + return Promise.resolve(); + } + if (!sourceAudioResumePromiseRef.current) { + sourceAudioResumePromiseRef.current = context + .resume() + .catch(() => undefined) + .finally(() => { + sourceAudioResumePromiseRef.current = null; + }); + } + return sourceAudioResumePromiseRef.current; + }; + + useEffect(() => { + let cancelled = false; + const existing = audioElementsRef.current; + const currentIds = new Set(resolvedUserTracks.map((track) => track.id)); + + for (const [id, audio] of existing) { + if (!currentIds.has(id)) { + audio.pause(); + audio.src = ""; + audioElementRevokersRef.current.get(id)?.(); + audioElementRevokersRef.current.delete(id); + audioElementResourcesRef.current.delete(id); + existing.delete(id); + } + } + + for (const track of resolvedUserTracks) { + let audio = existing.get(track.id); + if (!audio) { + audio = new Audio(); + audio.preload = "auto"; + existing.set(track.id, audio); + } + + if (audioElementResourcesRef.current.get(track.id) !== track.sourceRef.path) { + audio.pause(); + audio.src = ""; + audioElementRevokersRef.current.get(track.id)?.(); + audioElementRevokersRef.current.delete(track.id); + audioElementResourcesRef.current.set(track.id, track.sourceRef.path); + + void (async () => { + const resolved = await resolveMediaElementSource(track.sourceRef.path); + const latestAudio = existing.get(track.id); + + if ( + cancelled || + latestAudio !== audio || + audioElementResourcesRef.current.get(track.id) !== track.sourceRef.path + ) { + resolved.revoke(); + return; + } + + audioElementRevokersRef.current.set(track.id, resolved.revoke); + latestAudio.src = resolved.src; + })(); + } + + audio.volume = Math.max(0, Math.min(1, track.gain * previewVolume)); + } + + return () => { + cancelled = true; + }; + }, [previewVolume, resolvedUserTracks]); + + useEffect(() => { + let cancelled = false; + const existing = sourceAudioElementsRef.current; + const currentIds = new Set(resolvedSourceTracks.map((track) => track.sourceRef.path)); + + for (const [id, audio] of existing) { + if (!currentIds.has(id)) { + audio.pause(); + audio.src = ""; + sourceAudioMediaNodesRef.current.get(id)?.disconnect(); + sourceAudioMediaNodesRef.current.delete(id); + sourceAudioGainNodesRef.current.get(id)?.disconnect(); + sourceAudioGainNodesRef.current.delete(id); + sourceAudioElementRevokersRef.current.get(id)?.(); + sourceAudioElementRevokersRef.current.delete(id); + sourceAudioElementResourcesRef.current.delete(id); + existing.delete(id); + } + } + + for (const track of resolvedSourceTracks) { + const audioPath = track.sourceRef.path; + let audio = existing.get(audioPath); + if (!audio) { + audio = new Audio(); + audio.preload = "auto"; + audio.crossOrigin = "anonymous"; + existing.set(audioPath, audio); + } + audio.volume = 1; + audio.dataset.sourceAudioPath = audioPath; + + const context = ensureSourceAudioContext(); + const masterGain = sourceAudioMasterGainRef.current; + if (context && masterGain && !sourceAudioMediaNodesRef.current.has(audioPath)) { + try { + const mediaNode = context.createMediaElementSource(audio); + const trackGainNode = context.createGain(); + mediaNode.connect(trackGainNode); + trackGainNode.connect(masterGain); + sourceAudioMediaNodesRef.current.set(audioPath, mediaNode); + sourceAudioGainNodesRef.current.set(audioPath, trackGainNode); + } catch (error) { + onSourceFallbackLoadError(error); + } + } + + if (sourceAudioElementResourcesRef.current.get(audioPath) !== audioPath) { + audio.pause(); + audio.src = ""; + sourceAudioElementRevokersRef.current.get(audioPath)?.(); + sourceAudioElementRevokersRef.current.delete(audioPath); + sourceAudioElementResourcesRef.current.set(audioPath, audioPath); + + void (async () => { + try { + const resolved = await resolveMediaElementSource(audioPath); + const latestAudio = existing.get(audioPath); + + if ( + cancelled || + latestAudio !== audio || + sourceAudioElementResourcesRef.current.get(audioPath) !== audioPath + ) { + resolved.revoke(); + return; + } + + sourceAudioElementRevokersRef.current.set(audioPath, resolved.revoke); + latestAudio.src = resolved.src; + } catch (error) { + if (cancelled) { + return; + } + + sourceAudioElementRevokersRef.current.get(audioPath)?.(); + sourceAudioElementRevokersRef.current.delete(audioPath); + sourceAudioElementResourcesRef.current.delete(audioPath); + const latestAudio = existing.get(audioPath); + if (latestAudio === audio) { + latestAudio.pause(); + latestAudio.src = ""; + } + onSourceFallbackLoadError(error); + } + })(); + } + + const trackGainNode = sourceAudioGainNodesRef.current.get(audioPath); + if (trackGainNode) { + trackGainNode.gain.value = Math.max(0, Math.min(2, getSourceTrackPreviewGain(audioPath))); + } + } + + if (sourceAudioMasterGainRef.current) { + sourceAudioMasterGainRef.current.gain.value = isCurrentClipMuted + ? 0 + : Math.max(0, Math.min(1, previewVolume)); + } + + if (resolvedSourceTracks.length === 0) { + lastSourceAudioSyncTimeRef.current = null; + } + + return () => { + cancelled = true; + }; + }, [ + getSourceTrackPreviewGain, + isCurrentClipMuted, + onSourceFallbackLoadError, + resolvedSourceTracks, + previewVolume, + ]); + + useEffect(() => { + return () => { + for (const audio of audioElementsRef.current.values()) { + audio.pause(); + audio.src = ""; + } + for (const revoke of audioElementRevokersRef.current.values()) { + revoke(); + } + audioElementsRef.current.clear(); + audioElementRevokersRef.current.clear(); + audioElementResourcesRef.current.clear(); + for (const audio of sourceAudioElementsRef.current.values()) { + audio.pause(); + audio.src = ""; + } + for (const node of sourceAudioMediaNodesRef.current.values()) { + node.disconnect(); + } + for (const node of sourceAudioGainNodesRef.current.values()) { + node.disconnect(); + } + for (const revoke of sourceAudioElementRevokersRef.current.values()) { + revoke(); + } + sourceAudioElementsRef.current.clear(); + sourceAudioMediaNodesRef.current.clear(); + sourceAudioGainNodesRef.current.clear(); + sourceAudioElementRevokersRef.current.clear(); + sourceAudioElementResourcesRef.current.clear(); + if (sourceAudioMasterGainRef.current) { + sourceAudioMasterGainRef.current.disconnect(); + sourceAudioMasterGainRef.current = null; + } + const context = sourceAudioContextRef.current; + sourceAudioContextRef.current = null; + sourceAudioResumePromiseRef.current = null; + if (context) { + void context.close(); + } + lastSourceAudioSyncTimeRef.current = null; + }; + }, []); + + useEffect(() => { + const currentTimeMs = timelineTime * 1000; + const activeSpeedRegion = effectiveSpeedRegions.find( + (region) => currentTimeMs >= region.startMs && currentTimeMs < region.endMs, + ); + const targetPlaybackRate = activeSpeedRegion ? activeSpeedRegion.speed : 1; + + for (const track of resolvedUserTracks) { + const audio = audioElementsRef.current.get(track.id); + if (!audio) continue; + + const startMs = track.timelineBinding.startMs; + const endMs = track.timelineBinding.endMs; + const isInRegion = currentTimeMs >= startMs && currentTimeMs < endMs; + + if (isPlaying && isInRegion) { + enablePitchPreservingPlayback(audio); + const audioOffset = (currentTimeMs - startMs) / 1000; + if (Math.abs(audio.currentTime - audioOffset) > 0.2) { + audio.currentTime = audioOffset; + } + const syncedPlaybackRate = getMediaSyncPlaybackRate({ + basePlaybackRate: targetPlaybackRate, + currentTime: audio.currentTime, + targetTime: audioOffset, + }); + if (Math.abs(audio.playbackRate - syncedPlaybackRate) > 0.001) { + audio.playbackRate = syncedPlaybackRate; + } + if (audio.paused) { + audio.play().catch(() => undefined); + } + } else if (!audio.paused) { + audio.pause(); + } + } + }, [effectiveSpeedRegions, isPlaying, resolvedUserTracks, timelineTime]); + + useEffect(() => { + if (resolvedSourceTracks.length === 0) { + lastSourceAudioSyncTimeRef.current = null; + return; + } + + const activeSpeedRegion = effectiveSpeedRegions.find( + (region) => currentTime * 1000 >= region.startMs && currentTime * 1000 < region.endMs, + ); + const targetPlaybackRate = activeSpeedRegion ? activeSpeedRegion.speed : 1; + const previousTimelineTime = lastSourceAudioSyncTimeRef.current; + const timelineJumped = + previousTimelineTime === null || Math.abs(currentTime - previousTimelineTime) > 0.25; + const driftThreshold = isPlaying + ? SOURCE_AUDIO_PREVIEW_PLAYING_SEEK_DRIFT_SECONDS + : SOURCE_AUDIO_PREVIEW_PAUSED_SEEK_DRIFT_SECONDS; + if (sourceAudioMasterGainRef.current) { + sourceAudioMasterGainRef.current.gain.value = isCurrentClipMuted + ? 0 + : Math.max(0, Math.min(1, previewVolume)); + } + + for (const audio of sourceAudioElementsRef.current.values()) { + const sourceAudioPath = audio.dataset.sourceAudioPath ?? ""; + const trackGainNode = sourceAudioGainNodesRef.current.get(sourceAudioPath); + if (trackGainNode) { + trackGainNode.gain.value = Math.max(0, Math.min(2, getSourceTrackPreviewGain(sourceAudioPath))); + } + + enablePitchPreservingPlayback(audio); + const audioDuration = Number.isFinite(audio.duration) ? audio.duration : null; + const isMicCompanionTrack = /\.mic\./i.test(sourceAudioPath); + const rawStartDelaySeconds = estimateCompanionAudioStartDelaySeconds( + duration, + audioDuration, + sourceAudioFallbackStartDelayMsByPath[sourceAudioPath], + ); + const maxPreviewStartDelaySeconds = isMicCompanionTrack ? 2 : 5; + const startDelaySeconds = isMicCompanionTrack + ? 0 + : Number.isFinite(duration) && + (rawStartDelaySeconds >= Math.max(0, duration - 0.01) || + rawStartDelaySeconds > Math.max(maxPreviewStartDelaySeconds, duration * 0.9)) + ? 0 + : rawStartDelaySeconds; + const beforeAudioStart = currentTime + 0.001 < startDelaySeconds; + const targetTime = clampMediaTimeToDuration(currentTime - startDelaySeconds, audioDuration); + + const shouldSeek = + timelineJumped || + (!isPlaying && Math.abs(audio.currentTime - targetTime) > driftThreshold) || + (isPlaying && Math.abs(audio.currentTime - targetTime) > 0.9); + if (shouldSeek) { + try { + audio.currentTime = targetTime; + } catch { + // no-op + } + } + + // KISS for companion source tracks: fixed playback rate avoids audible flutter/stutter + // from continuous micro-corrections on system audio. + const syncedPlaybackRate = targetPlaybackRate; + if (Math.abs(audio.playbackRate - syncedPlaybackRate) > 0.001) { + audio.playbackRate = syncedPlaybackRate; + } + + const atEnd = audioDuration !== null && targetTime >= audioDuration; + if (isPlaying && !beforeAudioStart && !atEnd) { + void ensureSourceAudioRunning().then(() => { + audio.play().catch(() => undefined); + }); + } else if (!audio.paused) { + audio.pause(); + } + } + + lastSourceAudioSyncTimeRef.current = currentTime; + }, [ + currentTime, + duration, + effectiveSpeedRegions, + getSourceTrackPreviewGain, + isCurrentClipMuted, + isPlaying, + previewVolume, + resolvedSourceTracks, + sourceAudioFallbackStartDelayMsByPath, + ]); + + useEffect(() => { + if (!isPlaying || resolvedSourceTracks.length === 0) { + return; + } + void ensureSourceAudioRunning().then(() => { + for (const audio of sourceAudioElementsRef.current.values()) { + if (audio.paused) { + audio.play().catch(() => undefined); + } + } + }); + }, [isPlaying, resolvedSourceTracks.length]); +} diff --git a/src/components/video-editor/audio/useClipAudioSettingsController.ts b/src/components/video-editor/audio/useClipAudioSettingsController.ts new file mode 100644 index 00000000..08ad0d5e --- /dev/null +++ b/src/components/video-editor/audio/useClipAudioSettingsController.ts @@ -0,0 +1,93 @@ +import React, { useCallback, useMemo } from "react"; +import { + SOURCE_AUDIO_NORMALIZE_GAIN, + type SourceAudioTrackSettings, +} from "@/components/video-editor/audio/audioTypes"; +import { useSourceAudioTrackSettings } from "./useSourceAudioTrackSettings"; +import { getSourceTrackIdFromPath } from "@/lib/exporter/audioRoutingEngine"; + +interface UseClipAudioSettingsControllerParams { + selectedClipId: string | null; + activeClipId: string | null; + sourceAudioTrackSettingsByClip: Record; + setSourceAudioTrackSettingsByClip: React.Dispatch< + React.SetStateAction> + >; + defaultSourceAudioTrackSettings: SourceAudioTrackSettings; + setDefaultSourceAudioTrackSettings: React.Dispatch< + React.SetStateAction + >; +} + +export function useClipAudioSettingsController({ + selectedClipId, + activeClipId, + sourceAudioTrackSettingsByClip, + setSourceAudioTrackSettingsByClip, + defaultSourceAudioTrackSettings, + setDefaultSourceAudioTrackSettings, +}: UseClipAudioSettingsControllerParams) { + const { + sourceAudioTrackMeta, + activeSourceAudioTrackSettings, + selectedClipSourceAudioTrackSettings, + getSourceAudioTrackSettingsForClip, + onSourceAudioTracksMetaChange, + onSelectedClipSourceAudioTrackVolumeChange, + onSelectedClipSourceAudioTrackNormalizeChange, + } = useSourceAudioTrackSettings({ + selectedClipId, + activeClipId, + sourceAudioTrackSettingsByClip, + setSourceAudioTrackSettingsByClip, + defaultSourceAudioTrackSettings, + setDefaultSourceAudioTrackSettings, + }); + + const previewSourceAudioTrackSettings = useMemo( + () => + activeClipId ? activeSourceAudioTrackSettings : selectedClipSourceAudioTrackSettings, + [activeClipId, activeSourceAudioTrackSettings, selectedClipSourceAudioTrackSettings], + ); + + const embeddedTrackId = useMemo<"mixed" | "system">(() => { + const hasMixedTrack = sourceAudioTrackMeta.some((track) => track.id === "mixed"); + if (hasMixedTrack) return "mixed"; + const hasSystemTrack = sourceAudioTrackMeta.some((track) => track.id === "system"); + return hasSystemTrack ? "system" : "mixed"; + }, [sourceAudioTrackMeta]); + + const embeddedSourcePreviewGain = useMemo(() => { + const settings = previewSourceAudioTrackSettings[embeddedTrackId] ?? { + volume: 1, + normalize: false, + }; + const normalizeGain = settings.normalize ? SOURCE_AUDIO_NORMALIZE_GAIN : 1; + return Math.max(0, Math.min(2, settings.volume * normalizeGain)); + }, [embeddedTrackId, previewSourceAudioTrackSettings]); + + const getSourceTrackPreviewGain = useCallback( + (audioPath: string) => { + const trackId = getSourceTrackIdFromPath(audioPath); + const settings = previewSourceAudioTrackSettings[trackId] ?? { + volume: 1, + normalize: false, + }; + const normalizeGain = settings.normalize ? SOURCE_AUDIO_NORMALIZE_GAIN : 1; + return Math.max(0, Math.min(2, settings.volume * normalizeGain)); + }, + [previewSourceAudioTrackSettings], + ); + + return { + sourceAudioTrackMeta, + activeSourceAudioTrackSettings, + selectedClipSourceAudioTrackSettings, + getSourceAudioTrackSettingsForClip, + onSourceAudioTracksMetaChange, + onSelectedClipSourceAudioTrackVolumeChange, + onSelectedClipSourceAudioTrackNormalizeChange, + embeddedSourcePreviewGain, + getSourceTrackPreviewGain, + }; +} diff --git a/src/components/video-editor/audio/useSourceAudioFallback.ts b/src/components/video-editor/audio/useSourceAudioFallback.ts new file mode 100644 index 00000000..69be8c24 --- /dev/null +++ b/src/components/video-editor/audio/useSourceAudioFallback.ts @@ -0,0 +1,68 @@ +import { useEffect, useState } from "react"; +import { toast } from "sonner"; +import { SOURCE_AUDIO_FALLBACK_TOAST_ID } from "@/components/video-editor/audio/audioTypes"; + +interface UseSourceAudioFallbackParams { + currentSourcePath: string | null; + summarizeErrorMessage: (message: string) => string; +} + +export function useSourceAudioFallback({ + currentSourcePath, + summarizeErrorMessage, +}: UseSourceAudioFallbackParams) { + const [sourceAudioFallbackPaths, setSourceAudioFallbackPaths] = useState([]); + const [sourceAudioFallbackStartDelayMsByPath, setSourceAudioFallbackStartDelayMsByPath] = + useState>({}); + + useEffect(() => { + let cancelled = false; + setSourceAudioFallbackPaths([]); + setSourceAudioFallbackStartDelayMsByPath({}); + + if (!currentSourcePath) { + return () => { + cancelled = true; + }; + } + + void (async () => { + try { + const result = await window.electronAPI.getVideoAudioFallbackPaths(currentSourcePath); + if (cancelled) { + return; + } + if (!result.success) { + setSourceAudioFallbackPaths([]); + setSourceAudioFallbackStartDelayMsByPath({}); + toast.warning( + result.error + ? `Could not load companion audio sources: ${summarizeErrorMessage(result.error)}` + : "Could not load companion audio sources. Playback and export may miss microphone audio.", + { id: SOURCE_AUDIO_FALLBACK_TOAST_ID, duration: 10000 }, + ); + return; + } + + toast.dismiss(SOURCE_AUDIO_FALLBACK_TOAST_ID); + setSourceAudioFallbackPaths(result.paths ?? []); + setSourceAudioFallbackStartDelayMsByPath(result.startDelayMsByPath ?? {}); + } catch (error) { + if (!cancelled) { + setSourceAudioFallbackPaths([]); + setSourceAudioFallbackStartDelayMsByPath({}); + toast.warning( + `Could not load companion audio sources: ${summarizeErrorMessage(String(error))}`, + { id: SOURCE_AUDIO_FALLBACK_TOAST_ID, duration: 10000 }, + ); + } + } + })(); + + return () => { + cancelled = true; + }; + }, [currentSourcePath, summarizeErrorMessage]); + + return { sourceAudioFallbackPaths, sourceAudioFallbackStartDelayMsByPath }; +} diff --git a/src/components/video-editor/audio/useSourceAudioTrackSettings.ts b/src/components/video-editor/audio/useSourceAudioTrackSettings.ts new file mode 100644 index 00000000..22688534 --- /dev/null +++ b/src/components/video-editor/audio/useSourceAudioTrackSettings.ts @@ -0,0 +1,176 @@ +import React, { useCallback, useMemo, useState } from "react"; +import type { + SourceAudioTrackMeta, + SourceAudioTrackSettings, +} from "@/components/video-editor/audio/audioTypes"; + +interface UseSourceAudioTrackSettingsParams { + selectedClipId: string | null; + activeClipId: string | null; + sourceAudioTrackSettingsByClip: Record; + setSourceAudioTrackSettingsByClip: React.Dispatch< + React.SetStateAction> + >; + defaultSourceAudioTrackSettings: SourceAudioTrackSettings; + setDefaultSourceAudioTrackSettings: React.Dispatch>; +} + +export interface UseSourceAudioTrackSettingsResult { + sourceAudioTrackMeta: SourceAudioTrackMeta; + activeSourceAudioTrackSettings: SourceAudioTrackSettings; + selectedClipSourceAudioTrackSettings: SourceAudioTrackSettings; + getSourceAudioTrackSettingsForClip: (clipId: string | null) => SourceAudioTrackSettings; + onSourceAudioTracksMetaChange: (tracks: SourceAudioTrackMeta) => void; + onSelectedClipSourceAudioTrackVolumeChange: (id: string, volume: number) => void; + onSelectedClipSourceAudioTrackNormalizeChange: (id: string, normalize: boolean) => void; +} + +function isSameTrackMeta(left: SourceAudioTrackMeta, right: SourceAudioTrackMeta): boolean { + if (left.length !== right.length) return false; + for (let index = 0; index < left.length; index += 1) { + const leftTrack = left[index]; + const rightTrack = right[index]; + if (!leftTrack || !rightTrack) return false; + if (leftTrack.id !== rightTrack.id || leftTrack.label !== rightTrack.label) { + return false; + } + } + return true; +} + +export function useSourceAudioTrackSettings({ + selectedClipId, + activeClipId, + sourceAudioTrackSettingsByClip, + setSourceAudioTrackSettingsByClip, + defaultSourceAudioTrackSettings, + setDefaultSourceAudioTrackSettings, +}: UseSourceAudioTrackSettingsParams): UseSourceAudioTrackSettingsResult { + const [sourceAudioTrackMeta, setSourceAudioTrackMeta] = useState([]); + + const activeSourceAudioTrackSettings = useMemo(() => { + if (!activeClipId) { + return defaultSourceAudioTrackSettings; + } + return { + ...defaultSourceAudioTrackSettings, + ...(sourceAudioTrackSettingsByClip[activeClipId] ?? {}), + }; + }, [activeClipId, defaultSourceAudioTrackSettings, sourceAudioTrackSettingsByClip]); + + const selectedClipSourceAudioTrackSettings = useMemo(() => { + if (!selectedClipId) { + return defaultSourceAudioTrackSettings; + } + return { + ...defaultSourceAudioTrackSettings, + ...(sourceAudioTrackSettingsByClip[selectedClipId] ?? {}), + }; + }, [defaultSourceAudioTrackSettings, selectedClipId, sourceAudioTrackSettingsByClip]); + + const onSourceAudioTracksMetaChange = useCallback((tracks: SourceAudioTrackMeta) => { + setSourceAudioTrackMeta((prev) => (isSameTrackMeta(prev, tracks) ? prev : tracks)); + setDefaultSourceAudioTrackSettings((prev) => { + const next: SourceAudioTrackSettings = {}; + for (const track of tracks) { + next[track.id] = prev[track.id] ?? { volume: 1, normalize: false }; + } + const prevKeys = Object.keys(prev); + const nextKeys = Object.keys(next); + if (prevKeys.length !== nextKeys.length) { + return next; + } + for (const key of nextKeys) { + const prevSetting = prev[key]; + const nextSetting = next[key]; + if (!prevSetting || !nextSetting) { + return next; + } + if ( + prevSetting.volume !== nextSetting.volume || + prevSetting.normalize !== nextSetting.normalize + ) { + return next; + } + } + return prev; + }); + }, []); + + const getSourceAudioTrackSettingsForClip = useCallback( + (clipId: string | null): SourceAudioTrackSettings => { + if (!clipId) { + return defaultSourceAudioTrackSettings; + } + return { + ...defaultSourceAudioTrackSettings, + ...(sourceAudioTrackSettingsByClip[clipId] ?? {}), + }; + }, + [defaultSourceAudioTrackSettings, sourceAudioTrackSettingsByClip], + ); + + const onSelectedClipSourceAudioTrackVolumeChange = useCallback( + (id: string, volume: number) => { + if (!selectedClipId) return; + setSourceAudioTrackSettingsByClip((prev) => { + const prevClip = prev[selectedClipId] ?? defaultSourceAudioTrackSettings; + const nextVolume = Number.isFinite(volume) + ? Math.max(0, Math.min(2, volume)) + : (prevClip[id]?.volume ?? 1); + const prevNormalize = prevClip[id]?.normalize ?? false; + if ( + prevClip[id]?.volume === nextVolume && + prevClip[id]?.normalize === prevNormalize + ) { + return prev; + } + return { + ...prev, + [selectedClipId]: { + ...prevClip, + [id]: { + volume: nextVolume, + normalize: prevNormalize, + }, + }, + }; + }); + }, + [defaultSourceAudioTrackSettings, selectedClipId], + ); + + const onSelectedClipSourceAudioTrackNormalizeChange = useCallback( + (id: string, normalize: boolean) => { + if (!selectedClipId) return; + setSourceAudioTrackSettingsByClip((prev) => { + const prevClip = prev[selectedClipId] ?? defaultSourceAudioTrackSettings; + const prevVolume = prevClip[id]?.volume ?? 1; + if (prevClip[id]?.normalize === normalize) { + return prev; + } + return { + ...prev, + [selectedClipId]: { + ...prevClip, + [id]: { + volume: prevVolume, + normalize, + }, + }, + }; + }); + }, + [defaultSourceAudioTrackSettings, selectedClipId], + ); + + return { + sourceAudioTrackMeta, + activeSourceAudioTrackSettings, + selectedClipSourceAudioTrackSettings, + getSourceAudioTrackSettingsForClip, + onSourceAudioTracksMetaChange, + onSelectedClipSourceAudioTrackVolumeChange, + onSelectedClipSourceAudioTrackNormalizeChange, + }; +} diff --git a/src/components/video-editor/audio/useVideoEditorAudio.ts b/src/components/video-editor/audio/useVideoEditorAudio.ts new file mode 100644 index 00000000..a82ea941 --- /dev/null +++ b/src/components/video-editor/audio/useVideoEditorAudio.ts @@ -0,0 +1,148 @@ +import React, { useMemo } from "react"; +import { resolveSourceTrackRoutingPolicy } from "@/lib/exporter/sourceTrackRoutingPolicy"; +import type { + AudioRegion, + ClipRegion, + SpeedRegion, +} from "../types"; +import type { SourceAudioTrackSettings } from "@/components/video-editor/audio/audioTypes"; +import { getActiveClipIdAtSourceTime, isClipMutedById } from "./clipAudio"; +import { useAudioPreviewSync } from "./useAudioPreviewSync"; +import { useClipAudioSettingsController } from "./useClipAudioSettingsController"; +import { useSourceAudioFallback } from "./useSourceAudioFallback"; + +function extractLocalPathFromMediaServerUrl(input: string | null | undefined): string | null { + if (!input) return null; + try { + const url = new URL(input); + const isLocalMediaServer = + (url.protocol === "http:" || url.protocol === "https:") && + (url.hostname === "127.0.0.1" || url.hostname === "localhost") && + url.pathname === "/video"; + if (!isLocalMediaServer) return null; + return url.searchParams.get("path"); + } catch { + return null; + } +} + +interface UseVideoEditorAudioParams { + currentSourcePath: string | null; + selectedClipId: string | null; + clipRegions: ClipRegion[]; + audioRegions: AudioRegion[]; + effectiveSpeedRegions: SpeedRegion[]; + sourceAudioTrackSettingsByClip: Record; + setSourceAudioTrackSettingsByClip: React.Dispatch< + React.SetStateAction> + >; + defaultSourceAudioTrackSettings: SourceAudioTrackSettings; + setDefaultSourceAudioTrackSettings: React.Dispatch< + React.SetStateAction + >; + currentTime: number; + timelineTime: number; + duration: number; + isPlaying: boolean; + previewVolume: number; + summarizeErrorMessage: (message: string) => string; + onSourceFallbackLoadError: (error: unknown) => void; +} + +export function useVideoEditorAudio({ + currentSourcePath, + selectedClipId, + clipRegions, + audioRegions, + effectiveSpeedRegions, + sourceAudioTrackSettingsByClip, + setSourceAudioTrackSettingsByClip, + defaultSourceAudioTrackSettings, + setDefaultSourceAudioTrackSettings, + currentTime, + timelineTime, + duration, + isPlaying, + previewVolume, + summarizeErrorMessage, + onSourceFallbackLoadError, +}: UseVideoEditorAudioParams) { + const fallbackLookupSourcePath = useMemo( + () => extractLocalPathFromMediaServerUrl(currentSourcePath) ?? currentSourcePath, + [currentSourcePath], + ); + + const { sourceAudioFallbackPaths, sourceAudioFallbackStartDelayMsByPath } = + useSourceAudioFallback({ + currentSourcePath: fallbackLookupSourcePath, + summarizeErrorMessage, + }); + + const sourceTrackRoutingPolicy = useMemo( + () => resolveSourceTrackRoutingPolicy(currentSourcePath, sourceAudioFallbackPaths), + [currentSourcePath, sourceAudioFallbackPaths], + ); + const previewSourceAudioFallbackPaths = sourceTrackRoutingPolicy.playbackPaths; + const shouldMutePreviewVideo = sourceTrackRoutingPolicy.muteEmbeddedPreview; + + const activeClipIdAtCurrentTime = useMemo( + () => getActiveClipIdAtSourceTime(currentTime, clipRegions), + [clipRegions, currentTime], + ); + const isCurrentClipMuted = useMemo( + () => isClipMutedById(activeClipIdAtCurrentTime, clipRegions), + [activeClipIdAtCurrentTime, clipRegions], + ); + + const { + sourceAudioTrackMeta, + activeSourceAudioTrackSettings, + selectedClipSourceAudioTrackSettings, + getSourceAudioTrackSettingsForClip, + onSourceAudioTracksMetaChange, + onSelectedClipSourceAudioTrackVolumeChange, + onSelectedClipSourceAudioTrackNormalizeChange, + embeddedSourcePreviewGain, + getSourceTrackPreviewGain, + } = useClipAudioSettingsController({ + selectedClipId, + activeClipId: activeClipIdAtCurrentTime, + sourceAudioTrackSettingsByClip, + setSourceAudioTrackSettingsByClip, + defaultSourceAudioTrackSettings, + setDefaultSourceAudioTrackSettings, + }); + + useAudioPreviewSync({ + audioRegions, + previewVolume, + isPlaying, + currentTime, + timelineTime, + duration, + effectiveSpeedRegions, + previewSourceAudioFallbackPaths, + sourceAudioFallbackStartDelayMsByPath, + isCurrentClipMuted, + getSourceTrackPreviewGain, + onSourceFallbackLoadError, + }); + + return { + sourceAudioFallbackPaths, + sourceAudioFallbackStartDelayMsByPath, + previewSourceAudioFallbackPaths, + shouldMutePreviewVideo, + activeClipIdAtCurrentTime, + isCurrentClipMuted, + sourceAudioTrackMeta, + activeSourceAudioTrackSettings, + selectedClipSourceAudioTrackSettings, + getSourceAudioTrackSettingsForClip, + onSourceAudioTracksMetaChange, + onSelectedClipSourceAudioTrackVolumeChange, + onSelectedClipSourceAudioTrackNormalizeChange, + embeddedSourcePreviewGain, + getSourceTrackPreviewGain, + }; +} diff --git a/src/components/video-editor/audio/waveform/WaveformGenerator.ts b/src/components/video-editor/audio/waveform/WaveformGenerator.ts new file mode 100644 index 00000000..b44c274f --- /dev/null +++ b/src/components/video-editor/audio/waveform/WaveformGenerator.ts @@ -0,0 +1,113 @@ +import WorkerConstructor from "./waveform.worker?worker"; +import type { AudioPeaksData } from "../../timeline/core/timelineTypes"; +import { WAVEFORM_DEFAULT_PEAK_COUNT } from "../../timeline/core/constants"; + +export class WaveformGenerator { + private audioContext: AudioContext; + private worker: Worker; + private peaksCache = new Map(); + private pending = new Map>(); + private workerRequestSeq = 0; + private workerResolvers = new Map void; reject: (err: Error) => void }>(); + + constructor() { + this.audioContext = new (window.AudioContext || (window as typeof window & { webkitAudioContext?: typeof AudioContext }).webkitAudioContext)(); + this.worker = new WorkerConstructor(); + + this.worker.addEventListener( + "message", + (event: MessageEvent<{ requestId: number; peaks?: Float32Array; error?: string }>) => { + const { requestId, peaks, error } = event.data; + const resolver = this.workerResolvers.get(requestId); + if (!resolver) return; + + this.workerResolvers.delete(requestId); + if (error) { + resolver.reject(new Error(error)); + } else if (peaks) { + resolver.resolve(peaks); + } + }, + ); + + this.worker.addEventListener("error", (error: ErrorEvent) => { + console.error("[WaveformGenerator] Worker fatal error:", error); + const fatalError = error.error ?? new Error(error.message || "Worker crashed"); + + // Reject all pending requests if the worker itself crashes + for (const resolver of this.workerResolvers.values()) { + resolver.reject(fatalError); + } + this.workerResolvers.clear(); + }); + } + + private computePeaksWithWorker(channels: Float32Array[], samples: number): Promise { + return new Promise((resolve, reject) => { + const requestId = ++this.workerRequestSeq; + this.workerResolvers.set(requestId, { resolve, reject }); + + this.worker.postMessage( + { + requestId, + channels, + samples, + }, + channels.map(c => c.buffer), + ); + }); + } + + public async generate(url: string, peakCount = WAVEFORM_DEFAULT_PEAK_COUNT): Promise { + const cacheKey = `${url}::${peakCount}`; + const cached = this.peaksCache.get(cacheKey); + if (cached) return cached; + + const inflight = this.pending.get(cacheKey); + if (inflight) return inflight; + + const request = (async () => { + const response = await fetch(url); + if (!response.ok) { + throw new Error(`Failed to load media: ${response.status}`); + } + + const arrayBuffer = await response.arrayBuffer(); + const decoded = await this.audioContext.decodeAudioData(arrayBuffer); + + const channels: Float32Array[] = []; + for (let i = 0; i < decoded.numberOfChannels; i++) { + // We slice to transfer the underlying buffer to the worker + channels.push(decoded.getChannelData(i).slice()); + } + + const peaks = await this.computePeaksWithWorker(channels, peakCount); + + let max = 0; + for (let i = 0; i < peaks.length; i++) { + if (peaks[i] > max) max = peaks[i]; + } + if (max > 0) { + for (let i = 0; i < peaks.length; i++) { + peaks[i] /= max; + } + } + + const result: AudioPeaksData = { + peaks, + durationMs: decoded.duration * 1000, + }; + this.peaksCache.set(cacheKey, result); + this.pending.delete(cacheKey); + return result; + })().catch((error) => { + this.pending.delete(cacheKey); + throw error; + }); + + this.pending.set(cacheKey, request); + return request; + } +} + +export const waveformGenerator = new WaveformGenerator(); diff --git a/src/components/video-editor/audio/waveform/waveform.worker.ts b/src/components/video-editor/audio/waveform/waveform.worker.ts new file mode 100644 index 00000000..1e43df98 --- /dev/null +++ b/src/components/video-editor/audio/waveform/waveform.worker.ts @@ -0,0 +1,48 @@ +type WaveformWorkerRequest = { + requestId: number; + channels: Float32Array[]; + samples: number; +}; + +interface WorkerContext { + onmessage: (e: MessageEvent) => void; + postMessage: (message: any, transfer?: Transferable[]) => void; +} + +const workerScope = self as unknown as WorkerContext; + +workerScope.onmessage = (e: MessageEvent) => { + const { requestId, channels, samples } = e.data; + + if (!channels || channels.length === 0 || samples <= 0) { + const empty = new Float32Array(0); + workerScope.postMessage({ requestId, peaks: empty }, [empty.buffer]); + return; + } + + try { + const firstChannel = channels[0]; + const result = new Float32Array(samples); + const total = firstChannel.length; + + for (let i = 0; i < samples; i++) { + const start = Math.floor((i * total) / samples); + const end = Math.floor(((i + 1) * total) / samples); + let max = 0; + for (let j = start; j < end; j++) { + for (let c = 0; c < channels.length; c++) { + const val = Math.abs(channels[c][j]); + if (val > max) max = val; + } + } + result[i] = max; + } + + workerScope.postMessage({ requestId, peaks: result }, [result.buffer]); + } catch (err) { + workerScope.postMessage({ + requestId, + error: err instanceof Error ? err.message : "Unknown worker error", + }); + } +}; diff --git a/src/components/video-editor/projectPersistence.ts b/src/components/video-editor/projectPersistence.ts index a75deb78..9adb3608 100644 --- a/src/components/video-editor/projectPersistence.ts +++ b/src/components/video-editor/projectPersistence.ts @@ -20,6 +20,7 @@ import { import { DEFAULT_WALLPAPER_PATH } from "@/lib/wallpapers"; import { ASPECT_RATIOS, type AspectRatio, isCustomAspectRatio } from "@/utils/aspectRatioUtils"; import { CURSOR_MOTION_PRESETS, resolveCursorMotionPresetId } from "./cursorMotionPresets"; +import type { SourceAudioTrackSettings } from "@/components/video-editor/audio/audioTypes"; import { type AnnotationRegion, type AudioRegion, @@ -127,6 +128,8 @@ export interface ProjectEditorState { autoCaptionSettings: AutoCaptionSettings; webcam: WebcamOverlaySettings; aspectRatio: AspectRatio; + sourceAudioTrackSettingsByClip?: Record; + defaultSourceAudioTrackSettings?: SourceAudioTrackSettings; exportEncodingMode: ExportEncodingMode; exportBackendPreference: ExportBackendPreference; exportPipelineModel: ExportPipelineModel; @@ -496,6 +499,10 @@ export function normalizeProjectEditor(editor: Partial): Pro endMs, speed: isFiniteNumber(region.speed) ? region.speed : 1, muted: typeof region.muted === "boolean" ? region.muted : false, + showSourceAudio: + typeof region.showSourceAudio === "boolean" + ? region.showSourceAudio + : false, }; }) : []; @@ -647,16 +654,17 @@ export function normalizeProjectEditor(editor: Partial): Pro const startMs = Math.max(0, Math.min(rawStart, rawEnd)); const endMs = Math.max(startMs + 1, rawEnd); - return { - id: region.id, - startMs, - endMs, - audioPath: typeof region.audioPath === "string" ? region.audioPath : "", - volume: isFiniteNumber(region.volume) ? clamp(region.volume, 0, 1) : 1, - trackIndex: isFiniteNumber(region.trackIndex) - ? Math.max(0, Math.floor(region.trackIndex)) - : 0, - }; + return { + id: region.id, + startMs, + endMs, + audioPath: typeof region.audioPath === "string" ? region.audioPath : "", + volume: isFiniteNumber(region.volume) ? clamp(region.volume, 0, 1) : 1, + normalize: Boolean(region.normalize), + trackIndex: isFiniteNumber(region.trackIndex) + ? Math.max(0, Math.floor(region.trackIndex)) + : 0, + }; }) : []; @@ -983,6 +991,16 @@ export function normalizeProjectEditor(editor: Partial): Pro ? clamp(webcam.margin, 0, 96) : DEFAULT_WEBCAM_MARGIN, }, + sourceAudioTrackSettingsByClip: + editor.sourceAudioTrackSettingsByClip && + typeof editor.sourceAudioTrackSettingsByClip === "object" + ? editor.sourceAudioTrackSettingsByClip + : {}, + defaultSourceAudioTrackSettings: + editor.defaultSourceAudioTrackSettings && + typeof editor.defaultSourceAudioTrackSettings === "object" + ? editor.defaultSourceAudioTrackSettings + : {}, aspectRatio: typeof editor.aspectRatio === "string" && (validAspectRatios.has(editor.aspectRatio as AspectRatio) || diff --git a/src/components/video-editor/timeline/Item.tsx b/src/components/video-editor/timeline/Item.tsx index 333b8855..44405905 100644 --- a/src/components/video-editor/timeline/Item.tsx +++ b/src/components/video-editor/timeline/Item.tsx @@ -5,18 +5,22 @@ import { MusicNotes as Music, MouseLeftClickIcon as PhMouseLeftClick, Scissors, + SpeakerX, MagnifyingGlassPlus as ZoomIn, } from "@phosphor-icons/react"; import type { Span } from "dnd-timeline"; import { useItem } from "dnd-timeline"; import { useMemo } from "react"; import { cn } from "@/lib/utils"; +import AudioWaveform from "./components/waveform/AudioWaveform"; +import type { AudioPeaksData } from "./core/timelineTypes"; import glassStyles from "./ItemGlass.module.css"; interface ItemProps { id: string; span: Span; rowId: string; + disabled?: boolean; children: React.ReactNode; isSelected?: boolean; onSelect?: () => void; @@ -24,6 +28,11 @@ interface ItemProps { zoomDepth?: number; zoomMode?: "auto" | "manual"; speedValue?: number; + waveformPeaks?: AudioPeaksData | null; + waveformSegmentSpan?: Span; + waveformGain?: number; + waveformNormalize?: boolean; + muted?: boolean; variant?: "zoom" | "trim" | "clip" | "annotation" | "speed" | "audio"; } @@ -51,18 +60,25 @@ export default function Item({ id, span, rowId, + disabled = false, isSelected = false, onSelect, onSelectId, zoomDepth = 1, zoomMode = "auto", speedValue, + waveformPeaks = null, + waveformSegmentSpan, + waveformGain = 1, + waveformNormalize = false, + muted = false, variant = "zoom", children, }: ItemProps) { const { setNodeRef, attributes, listeners, itemStyle, itemContentStyle } = useItem({ id, span, + disabled, data: { rowId }, }); @@ -71,6 +87,7 @@ export default function Item({ const isClip = variant === "clip"; const isSpeed = variant === "speed"; const isAudio = variant === "audio"; + const showAudioWaveform = isAudio && Boolean(waveformPeaks); const glassClass = isZoom ? glassStyles.glassPurple @@ -146,6 +163,22 @@ export default function Item({ style={{ cursor: "col-resize", pointerEvents: "auto" }} title="Resize right" /> + {showAudioWaveform && waveformPeaks && ( + + )} + {/* Muted overlay for source audio track items */} + {isAudio && muted && ( +
    + +
    + )} {/* Content */}
    diff --git a/src/components/video-editor/timeline/TimelineEditor.tsx b/src/components/video-editor/timeline/TimelineEditor.tsx index 818caa42..df2ed47b 100644 --- a/src/components/video-editor/timeline/TimelineEditor.tsx +++ b/src/components/video-editor/timeline/TimelineEditor.tsx @@ -17,6 +17,12 @@ import { } from "@/utils/aspectRatioUtils"; import { formatShortcut } from "@/utils/platformUtils"; import { loadEditorPreferences, saveEditorPreferences } from "../editorPreferences"; +import { fromFileUrl } from "../projectPersistence"; +import type { + SourceAudioTrackMeta, + SourceAudioTrackSettings, + SourceAudioTrackWithPeaks, +} from "@/components/video-editor/audio/audioTypes"; import type { AnnotationRegion, AudioRegion, @@ -80,6 +86,38 @@ export interface TimelineEditorProps { isCropped?: boolean; videoPath?: string | null; hideToolbar?: boolean; + showSourceAudioTrack?: boolean; + onSourceAudioAvailabilityChange?: (available: boolean) => void; + sourceAudioTrackSettings?: SourceAudioTrackSettings; + getSourceAudioTrackSettingsForClip?: ( + clipId: string | null, + ) => SourceAudioTrackSettings; + onSourceAudioTracksMetaChange?: (tracks: SourceAudioTrackMeta) => void; +} + +function extractLocalPathFromMediaServerUrl(input: string | null | undefined): string | null { + if (!input) return null; + try { + const url = new URL(input); + const isLocalMediaServer = + (url.protocol === "http:" || url.protocol === "https:") && + (url.hostname === "127.0.0.1" || url.hostname === "localhost") && + url.pathname === "/video"; + if (!isLocalMediaServer) return null; + return url.searchParams.get("path"); + } catch { + return null; + } +} + +function buildSourceSidecarPath(source: string, suffix: "mic" | "system"): string { + const normalized = source.replace(/\\/g, "/"); + const lastSlash = normalized.lastIndexOf("/"); + const dir = lastSlash >= 0 ? normalized.slice(0, lastSlash + 1) : ""; + const fileName = lastSlash >= 0 ? normalized.slice(lastSlash + 1) : normalized; + const dotIndex = fileName.lastIndexOf("."); + const baseName = dotIndex > 0 ? fileName.slice(0, dotIndex) : fileName; + return `${dir}${baseName}.${suffix}.wav`; } export interface TimelineEditorHandle { @@ -138,6 +176,11 @@ const TimelineEditor = forwardRef( isCropped = false, videoPath, hideToolbar = false, + showSourceAudioTrack = false, + onSourceAudioAvailabilityChange, + sourceAudioTrackSettings = {}, + getSourceAudioTrackSettingsForClip, + onSourceAudioTracksMetaChange, }, ref, ) { @@ -178,8 +221,111 @@ const TimelineEditor = forwardRef( pan: "Shift + Ctrl + Scroll", zoom: "Ctrl + Scroll", }); + const [liveSpanPreviewById, setLiveSpanPreviewById] = useState>({}); + const liveZoomPreview = useMemo(() => { + const previewSpans: Record = { ...liveSpanPreviewById }; + const hiddenZoomIds = new Set(); + + for (const [previewId, previewSpan] of Object.entries(liveSpanPreviewById)) { + const oldClip = clipRegions.find((clip) => clip.id === previewId); + if (!oldClip) continue; + + const newStart = Math.round(previewSpan.start); + const newEnd = Math.round(previewSpan.end); + const removedSegments = [ + ...(newStart > oldClip.startMs + ? [{ startMs: oldClip.startMs, endMs: newStart }] + : []), + ...(newEnd < oldClip.endMs + ? [{ startMs: newEnd, endMs: oldClip.endMs }] + : []), + ]; + + const startDelta = newStart - oldClip.startMs; + const endDelta = newEnd - oldClip.endMs; + const isMove = Math.abs(startDelta - endDelta) < 1 && Math.abs(startDelta) > 0; + + if (isMove) { + const delta = startDelta; + for (const zoom of zoomRegions) { + const overlaps = + zoom.startMs < oldClip.endMs && zoom.endMs > oldClip.startMs; + if (!overlaps) continue; + previewSpans[zoom.id] = { + start: zoom.startMs + delta, + end: zoom.endMs + delta, + }; + } + } + + if (removedSegments.length > 0) { + for (const zoom of zoomRegions) { + const removed = removedSegments.some( + (segment) => + zoom.startMs < segment.endMs && zoom.endMs > segment.startMs, + ); + if (removed) hiddenZoomIds.add(zoom.id); + } + } + } + + return { previewSpans, hiddenZoomIds }; + }, [clipRegions, liveSpanPreviewById, zoomRegions]); const { shortcuts: keyShortcuts, isMac } = useShortcuts(); - const audioPeaks = useTimelineAudioPeaks(videoPath); + const sourceAudioPeaks = useTimelineAudioPeaks(videoPath, { + enableSourceSidecarFallback: true, + }); + const localSourcePath = useMemo(() => { + if (!videoPath) return null; + return ( + extractLocalPathFromMediaServerUrl(videoPath) || + (/^file:\/\//i.test(videoPath) ? fromFileUrl(videoPath) : videoPath) + ); + }, [videoPath]); + const micSidecarPath = useMemo( + () => (localSourcePath ? buildSourceSidecarPath(localSourcePath, "mic") : null), + [localSourcePath], + ); + const systemSidecarPath = useMemo( + () => (localSourcePath ? buildSourceSidecarPath(localSourcePath, "system") : null), + [localSourcePath], + ); + const micSidecarPeaks = useTimelineAudioPeaks(micSidecarPath); + const systemSidecarPeaks = useTimelineAudioPeaks(systemSidecarPath); + const sourceAudioTracks = useMemo(() => { + if (systemSidecarPeaks || micSidecarPeaks) { + const tracks: SourceAudioTrackWithPeaks[] = []; + if (systemSidecarPeaks) + tracks.push({ + id: "system", + label: t("audio.systemLabel", "Source System"), + peaks: systemSidecarPeaks, + }); + if (micSidecarPeaks) + tracks.push({ + id: "mic", + label: t("audio.micLabel", "Source Mic"), + peaks: micSidecarPeaks, + }); + return tracks; + } + return sourceAudioPeaks + ? [ + { + id: "mixed", + label: t("audio.mixedLabel", "Source"), + peaks: sourceAudioPeaks, + }, + ] + : []; + }, [micSidecarPeaks, sourceAudioPeaks, systemSidecarPeaks, t]); + useEffect(() => { + onSourceAudioTracksMetaChange?.(sourceAudioTracks.map((t) => ({ id: t.id, label: t.label }))); + }, [onSourceAudioTracksMetaChange, sourceAudioTracks]); + void sourceAudioTrackSettings; + useEffect(() => { + onSourceAudioAvailabilityChange?.(sourceAudioTracks.length > 0); + }, [onSourceAudioAvailabilityChange, sourceAudioTracks.length]); useEffect(() => { if (aspectRatio === "native") { @@ -376,6 +522,25 @@ const TimelineEditor = forwardRef( onItemSpanChange={handleItemSpanChange} resolveTargetRowId={getResolvedDropRowId} allRegionSpans={allRegionSpans} + onLiveSpanPreviewChange={(id, span) => { + setLiveSpanPreviewById((prev) => { + if (!span) { + if (!(id in prev)) return prev; + const next = { ...prev }; + delete next[id]; + return next; + } + const current = prev[id]; + if ( + current && + current.start === span.start && + current.end === span.end + ) { + return prev; + } + return { ...prev, [id]: span }; + }); + }} > ( selectAllBlocksActive={selectAllBlocksActive} onClearBlockSelection={clearSelectedBlocks} keyframes={keyframes} - audioPeaks={audioPeaks} + sourceAudioTracks={sourceAudioTracks} + getSourceAudioTrackSettingsForClip={getSourceAudioTrackSettingsForClip} + showSourceAudioTrack={showSourceAudioTrack} + liveSpanPreviewById={liveZoomPreview.previewSpans} + liveHiddenItemIds={Array.from(liveZoomPreview.hiddenZoomIds)} />
    diff --git a/src/components/video-editor/timeline/components/viewport/TimelineCanvas.tsx b/src/components/video-editor/timeline/components/viewport/TimelineCanvas.tsx index 48e29b6f..aab23f76 100644 --- a/src/components/video-editor/timeline/components/viewport/TimelineCanvas.tsx +++ b/src/components/video-editor/timeline/components/viewport/TimelineCanvas.tsx @@ -11,18 +11,21 @@ import { type MouseEventHandler, } from "react"; import { cn } from "@/lib/utils"; +import type { + SourceAudioTrackSettings, + SourceAudioTrackWithPeaks, +} from "@/components/video-editor/audio/audioTypes"; import { getTimelineContentMinHeightPx, getTimelineRowsMinHeightPx, getTimelineViewportStretchFactor, TIMELINE_AXIS_HEIGHT_PX, } from "../../timelineLayout"; -import AudioWaveform from "../waveform/AudioWaveform"; import glassStyles from "../../ItemGlass.module.css"; import Item from "../../Item"; import Row from "../../Row"; -import { CLIP_ROW_ID, ZOOM_ROW_ID } from "../../core/constants"; -import type { AudioPeaksData, TimelineRenderItem } from "../../core/timelineTypes"; +import { CLIP_ROW_ID, SOURCE_AUDIO_ROW_ID, ZOOM_ROW_ID } from "../../core/constants"; +import type { TimelineRenderItem } from "../../core/timelineTypes"; import { getAnnotationTrackIndex, getAnnotationTrackRowId, @@ -34,6 +37,7 @@ import { import TimelineAxis from "../axis/TimelineAxis"; import ClipMarkerOverlay from "../overlays/ClipMarkerOverlay"; import PlaybackCursor from "../playhead/PlaybackCursor"; +import { useTimelineAudioPeaks } from "../../hooks/useTimelineAudioPeaks"; const HINT_CLIP = "Press C to split clip"; const HINT_ANNOTATION = "Press A to add annotation"; @@ -57,7 +61,13 @@ interface TimelineCanvasProps { selectAllBlocksActive?: boolean; onClearBlockSelection?: () => void; keyframes?: { id: string; time: number }[]; - audioPeaks?: AudioPeaksData | null; + sourceAudioTracks?: SourceAudioTrackWithPeaks[]; + getSourceAudioTrackSettingsForClip?: ( + clipId: string | null, + ) => SourceAudioTrackSettings; + showSourceAudioTrack?: boolean; + liveSpanPreviewById?: Record; + liveHiddenItemIds?: string[]; } interface TimelineHoverParams { @@ -218,7 +228,13 @@ interface TimelineCanvasRowsProps { onSelectClip?: (id: string | null) => void; onSelectAnnotation?: (id: string | null) => void; onSelectAudio?: (id: string | null) => void; - audioPeaks?: AudioPeaksData | null; + sourceAudioTracks?: SourceAudioTrackWithPeaks[]; + getSourceAudioTrackSettingsForClip?: ( + clipId: string | null, + ) => SourceAudioTrackSettings; + showSourceAudioTrack?: boolean; + liveSpanPreviewById?: Record; + liveHiddenItemIds?: string[]; direction: string; canShowGhostZoom: boolean; ghostStartMs: number | null; @@ -230,6 +246,44 @@ interface TimelineCanvasRowsProps { onZoomRowClick: MouseEventHandler; } +interface AudioItemWithWaveformProps { + item: TimelineRenderItem; + span: { start: number; end: number }; + waveformSpan: { start: number; end: number }; + isSelected: boolean; + onSelectAudio?: (id: string | null) => void; +} + +function AudioItemWithWaveform({ + item, + span, + waveformSpan, + isSelected, + onSelectAudio, +}: AudioItemWithWaveformProps) { + const peaks = useTimelineAudioPeaks(item.audioPath ?? null); + const normalizedWaveformSpan = useMemo(() => { + const duration = Math.max(0, waveformSpan.end - waveformSpan.start); + return { start: 0, end: duration }; + }, [waveformSpan.end, waveformSpan.start]); + return ( + + {item.label} + + ); +} + const TimelineCanvasRows = memo(function TimelineCanvasRows({ items, videoDurationMs, @@ -242,7 +296,11 @@ const TimelineCanvasRows = memo(function TimelineCanvasRows({ onSelectClip, onSelectAnnotation, onSelectAudio, - audioPeaks, + sourceAudioTracks = [], + getSourceAudioTrackSettingsForClip, + showSourceAudioTrack = false, + liveSpanPreviewById, + liveHiddenItemIds, direction, canShowGhostZoom, ghostStartMs, @@ -253,6 +311,7 @@ const TimelineCanvasRows = memo(function TimelineCanvasRows({ onZoomRowMouseLeave, onZoomRowClick, }: TimelineCanvasRowsProps) { + const hiddenIds = useMemo(() => new Set(liveHiddenItemIds ?? []), [liveHiddenItemIds]); const { clipItems, zoomItems, annotationRows, audioRows } = useMemo(() => { const nextClipItems: TimelineRenderItem[] = []; const nextZoomItems: TimelineRenderItem[] = []; @@ -307,7 +366,6 @@ const TimelineCanvasRows = memo(function TimelineCanvasRows({ return ( <> - {audioPeaks && } {clipItems.map((item) => ( ))} + {showSourceAudioTrack && + sourceAudioTracks.map((track) => ( + + {clipItems.filter(item => item.showSourceAudio).map((item) => { + const settings = getSourceAudioTrackSettingsForClip?.(item.id)?.[ + track.id + ] ?? { volume: 1, normalize: false }; + return ( + onSelectClip?.(item.id)} + variant="audio" + waveformPeaks={track.peaks} + waveformSegmentSpan={item.sourceSpan ?? item.span} + waveformGain={Math.max(0, Math.min(2, settings.volume))} + waveformNormalize={Boolean(settings.normalize)} + muted={item.muted} + > + {track.label} + + ); + })} + + ))}
    )} - {zoomItems.map((item) => ( + {zoomItems + .filter((item) => !hiddenIds.has(item.id)) + .map((item) => ( ( {rowItems.map((item) => ( - - {item.label} - + onSelectAudio={onSelectAudio} + /> ))} ))} @@ -431,7 +517,11 @@ export default function TimelineCanvas({ selectAllBlocksActive = false, onClearBlockSelection, keyframes = [], - audioPeaks, + sourceAudioTracks = [], + getSourceAudioTrackSettingsForClip, + showSourceAudioTrack = false, + liveSpanPreviewById, + liveHiddenItemIds, }: TimelineCanvasProps) { const { setTimelineRef, style, sidebarWidth, direction, range, valueToPixels, pixelsToValue } = useTimelineContext(); @@ -583,8 +673,9 @@ export default function TimelineCanvas({ if (isAnnotationTrackRowId(item.rowId)) annotationRowIds.add(item.rowId); if (isAudioTrackRowId(item.rowId)) audioRowIds.add(item.rowId); } - return 2 + annotationRowIds.size + audioRowIds.size; - }, [items]); + const sourceAudioRows = showSourceAudioTrack ? sourceAudioTracks.length : 0; + return 2 + sourceAudioRows + annotationRowIds.size + audioRowIds.size; + }, [items, showSourceAudioTrack, sourceAudioTracks.length]); const timelineRowsMinHeightPx = getTimelineRowsMinHeightPx(timelineRowCount); const timelineContentMinHeightPx = getTimelineContentMinHeightPx(timelineRowCount); const timelineViewportStretchFactor = getTimelineViewportStretchFactor(timelineRowCount); @@ -660,7 +751,11 @@ export default function TimelineCanvas({ onSelectClip={onSelectClip} onSelectAnnotation={onSelectAnnotation} onSelectAudio={onSelectAudio} - audioPeaks={audioPeaks} + sourceAudioTracks={sourceAudioTracks} + getSourceAudioTrackSettingsForClip={getSourceAudioTrackSettingsForClip} + showSourceAudioTrack={showSourceAudioTrack} + liveSpanPreviewById={liveSpanPreviewById} + liveHiddenItemIds={liveHiddenItemIds} direction={direction} canShowGhostZoom={canShowGhostZoom} ghostStartMs={ghostStartMs} diff --git a/src/components/video-editor/timeline/components/waveform/AudioWaveform.tsx b/src/components/video-editor/timeline/components/waveform/AudioWaveform.tsx index def1d607..2380d598 100644 --- a/src/components/video-editor/timeline/components/waveform/AudioWaveform.tsx +++ b/src/components/video-editor/timeline/components/waveform/AudioWaveform.tsx @@ -4,6 +4,11 @@ import type { AudioPeaksData } from "../../core/timelineTypes"; interface AudioWaveformProps { peaks: AudioPeaksData; + segmentStartMs?: number; + segmentEndMs?: number; + gain?: number; + normalize?: boolean; + className?: string; } /** @@ -11,10 +16,18 @@ interface AudioWaveformProps { * Automatically syncs with the timeline's visible range so the waveform * scrolls and zooms together with the clip items above it. */ -function AudioWaveformComponent({ peaks }: AudioWaveformProps) { +function AudioWaveformComponent({ + peaks, + segmentStartMs, + segmentEndMs, + gain = 1, + normalize = false, + className, +}: AudioWaveformProps) { const canvasRef = useRef(null); const { range } = useTimelineContext(); const [resizeKey, setResizeKey] = useState(0); + const lastDrawAtRef = useRef(0); // Bump resizeKey when the canvas element changes size. const observerRef = useRef(null); @@ -34,55 +47,77 @@ function AudioWaveformComponent({ peaks }: AudioWaveformProps) { useEffect(() => { const canvas = canvasRef.current; if (!canvas) return; + let rafId = 0; - const ctx = canvas.getContext("2d"); - if (!ctx) return; + const draw = () => { + const now = performance.now(); + if (now - lastDrawAtRef.current < 33) { + rafId = requestAnimationFrame(draw); + return; + } + lastDrawAtRef.current = now; - const rect = canvas.getBoundingClientRect(); - const dpr = window.devicePixelRatio || 1; - const width = Math.round(rect.width * dpr); - const height = Math.round(rect.height * dpr); + const ctx = canvas.getContext("2d"); + if (!ctx) return; - if (width === 0 || height === 0) return; + const rect = canvas.getBoundingClientRect(); + const dpr = window.devicePixelRatio || 1; + const width = Math.round(rect.width * dpr); + const height = Math.round(rect.height * dpr); - canvas.width = width; - canvas.height = height; + if (width === 0 || height === 0) return; - ctx.clearRect(0, 0, width, height); + canvas.width = width; + canvas.height = height; - const { peaks: peakData, durationMs } = peaks; - if (durationMs <= 0 || peakData.length === 0) return; + ctx.clearRect(0, 0, width, height); - const visibleStartMs = range.start; - const visibleEndMs = range.end; - const visibleDurationMs = visibleEndMs - visibleStartMs; - if (visibleDurationMs <= 0) return; + const { peaks: peakData, durationMs } = peaks; + if (durationMs <= 0 || peakData.length === 0) return; - const midY = height / 2; + const rawVisibleStartMs = segmentStartMs ?? range.start; + const rawVisibleEndMs = segmentEndMs ?? range.end; + const msPerBin = durationMs / peakData.length; + const visibleStartMs = + msPerBin > 0 ? Math.round(rawVisibleStartMs / msPerBin) * msPerBin : rawVisibleStartMs; + const visibleEndMs = + msPerBin > 0 ? Math.round(rawVisibleEndMs / msPerBin) * msPerBin : rawVisibleEndMs; + const visibleDurationMs = visibleEndMs - visibleStartMs; + if (visibleDurationMs <= 0) return; - ctx.beginPath(); - for (let px = 0; px < width; px++) { - const t = visibleStartMs + (px / width) * visibleDurationMs; - const binIndex = Math.min( - peakData.length - 1, - Math.max(0, Math.floor((t / durationMs) * peakData.length)), - ); - const amplitude = peakData[binIndex]; - const barHeight = amplitude * midY * 0.85; + const midY = height / 2; - ctx.moveTo(px, midY - barHeight); - ctx.lineTo(px, midY + barHeight); - } + ctx.beginPath(); + for (let px = 0; px < width; px++) { + const t = visibleStartMs + (px / width) * visibleDurationMs; + const exactIndex = Math.max( + 0, + Math.min(peakData.length - 1, (t / durationMs) * (peakData.length - 1)), + ); + const leftIndex = Math.floor(exactIndex); + const rightIndex = Math.min(peakData.length - 1, leftIndex + 1); + const mix = exactIndex - leftIndex; + let amplitude = peakData[leftIndex] * (1 - mix) + peakData[rightIndex] * mix; + if (normalize) amplitude = Math.sqrt(Math.max(0, amplitude)); + amplitude = Math.max(0, Math.min(1, amplitude * gain)); + const barHeight = amplitude * midY * 0.85; - ctx.strokeStyle = "rgba(255, 255, 255, 0.55)"; - ctx.lineWidth = dpr; - ctx.stroke(); - }, [peaks, range.start, range.end, resizeKey]); + ctx.moveTo(px, midY - barHeight); + ctx.lineTo(px, midY + barHeight); + } + + ctx.strokeStyle = "rgba(255, 255, 255, 0.55)"; + ctx.lineWidth = dpr; + ctx.stroke(); + }; + rafId = requestAnimationFrame(draw); + return () => cancelAnimationFrame(rafId); + }, [gain, normalize, peaks, range.start, range.end, resizeKey, segmentStartMs, segmentEndMs]); return ( ); diff --git a/src/components/video-editor/timeline/components/wrapper/TimelineWrapper.tsx b/src/components/video-editor/timeline/components/wrapper/TimelineWrapper.tsx index 5e777e70..f09b4001 100644 --- a/src/components/video-editor/timeline/components/wrapper/TimelineWrapper.tsx +++ b/src/components/video-editor/timeline/components/wrapper/TimelineWrapper.tsx @@ -29,6 +29,7 @@ interface TimelineWrapperProps { onItemSpanChange: (id: string, span: Span, rowId?: string) => void; resolveTargetRowId?: (id: string, proposedRowId: string) => string; allRegionSpans?: TimelineRegionSpan[]; + onLiveSpanPreviewChange?: (id: string, span: Span | null) => void; } export default function TimelineWrapper({ @@ -43,6 +44,7 @@ export default function TimelineWrapper({ onItemSpanChange, resolveTargetRowId, allRegionSpans = [], + onLiveSpanPreviewChange, }: TimelineWrapperProps) { const totalMs = Math.max(0, Math.round(videoDuration * 1000)); @@ -144,8 +146,12 @@ export default function TimelineWrapper({ ? (event.activatorEvent as PointerEvent).clientX + (event.delta?.x ?? 0) : undefined; if (span) showTooltip(span, screenX); + const moved = Math.hypot(event.delta?.x ?? 0, event.delta?.y ?? 0) > 0.01; + if (moved) { + onLiveSpanPreviewChange?.(event.active.id as string, span ?? null); + } }, - [showTooltip], + [onLiveSpanPreviewChange, showTooltip], ); const onResizeMove = useCallback( @@ -156,8 +162,9 @@ export default function TimelineWrapper({ ? (event.activatorEvent as PointerEvent).clientX + (event.delta?.x ?? 0) : undefined; if (span) showTooltip(span, screenX); + onLiveSpanPreviewChange?.(event.active.id as string, span ?? null); }, - [showTooltip], + [onLiveSpanPreviewChange, showTooltip], ); const hideTooltip = useCallback(() => showTooltip(null), [showTooltip]); @@ -166,16 +173,18 @@ export default function TimelineWrapper({ (event: ResizeEndEvent) => { hideTooltip(); onResizeEnd(event); + onLiveSpanPreviewChange?.(event.active.id as string, null); }, - [hideTooltip, onResizeEnd], + [hideTooltip, onLiveSpanPreviewChange, onResizeEnd], ); const onDragEndWithTooltip = useCallback( (event: DragEndEvent) => { hideTooltip(); onDragEnd(event); + onLiveSpanPreviewChange?.(event.active.id as string, null); }, - [hideTooltip, onDragEnd], + [hideTooltip, onDragEnd, onLiveSpanPreviewChange], ); const handleRangeChange = useCallback( diff --git a/src/components/video-editor/timeline/core/constants.ts b/src/components/video-editor/timeline/core/constants.ts index 1b71e996..88acba23 100644 --- a/src/components/video-editor/timeline/core/constants.ts +++ b/src/components/video-editor/timeline/core/constants.ts @@ -2,8 +2,10 @@ export const ZOOM_ROW_ID = "row-zoom"; export const CLIP_ROW_ID = "row-clip"; export const ANNOTATION_ROW_ID = "row-annotation"; export const AUDIO_ROW_ID = "row-audio"; +export const SOURCE_AUDIO_ROW_ID = "row-source-audio"; export const ANNOTATION_ROW_PREFIX = `${ANNOTATION_ROW_ID}-`; export const AUDIO_ROW_PREFIX = `${AUDIO_ROW_ID}-`; export const FALLBACK_RANGE_MS = 1000; export const TARGET_MARKER_COUNT = 12; +export const WAVEFORM_DEFAULT_PEAK_COUNT = 2048; diff --git a/src/components/video-editor/timeline/core/timelineTypes.ts b/src/components/video-editor/timeline/core/timelineTypes.ts index 2222f7cc..d1f705d7 100644 --- a/src/components/video-editor/timeline/core/timelineTypes.ts +++ b/src/components/video-editor/timeline/core/timelineTypes.ts @@ -31,10 +31,16 @@ export interface TimelineRenderItem { id: string; rowId: string; span: Span; + sourceSpan?: Span; label: string; + audioPath?: string; + audioGain?: number; + audioNormalize?: boolean; zoomDepth?: number; zoomMode?: ZoomMode; speedValue?: number; + showSourceAudio?: boolean; + muted?: boolean; variant: "zoom" | "trim" | "clip" | "annotation" | "speed" | "audio"; } diff --git a/src/components/video-editor/timeline/hooks/useTimelineAudioPeaks.ts b/src/components/video-editor/timeline/hooks/useTimelineAudioPeaks.ts index ed377663..fa750e29 100644 --- a/src/components/video-editor/timeline/hooks/useTimelineAudioPeaks.ts +++ b/src/components/video-editor/timeline/hooks/useTimelineAudioPeaks.ts @@ -1,81 +1,100 @@ import { useEffect, useRef, useState } from "react"; +import { resolveMediaResourceUrl } from "@/lib/exporter/localMediaSource"; +import { fromFileUrl } from "../../projectPersistence"; +import { waveformGenerator } from "../../audio/waveform/WaveformGenerator"; +import { WAVEFORM_DEFAULT_PEAK_COUNT } from "../core/constants"; import type { AudioPeaksData } from "../core/timelineTypes"; -/** Number of peak bins to produce — enough for smooth display at any zoom. */ -const TARGET_PEAK_COUNT = 2048; +function buildSidecarAudioCandidates(sourcePath: string): string[] { + const normalized = sourcePath.replace(/\\/g, "/"); + const lastSlash = normalized.lastIndexOf("/"); + const dir = lastSlash >= 0 ? normalized.slice(0, lastSlash + 1) : ""; + const fileName = lastSlash >= 0 ? normalized.slice(lastSlash + 1) : normalized; + const dotIndex = fileName.lastIndexOf("."); + const baseName = dotIndex > 0 ? fileName.slice(0, dotIndex) : fileName; -/** - * Decode audio from a media file URL and produce a fixed-length array of peak - * amplitudes suitable for waveform visualisation. - * - * Returns `null` while loading or if the file has no decodeable audio. - */ -export function useTimelineAudioPeaks(fileUrl: string | null | undefined): AudioPeaksData | null { + return [ + `${dir}${baseName}.system.wav`, + `${dir}${baseName}.mic.wav`, + `${dir}${baseName}.system.m4a`, + `${dir}${baseName}.mic.m4a`, + ]; +} + +function extractLocalPathFromMediaServerUrl(input: string): string | null { + try { + const url = new URL(input); + const isLocalMediaServer = + (url.protocol === "http:" || url.protocol === "https:") && + (url.hostname === "127.0.0.1" || url.hostname === "localhost") && + url.pathname === "/video"; + if (!isLocalMediaServer) return null; + return url.searchParams.get("path"); + } catch { + return null; + } +} + +interface TimelineAudioPeaksOptions { + enableSourceSidecarFallback?: boolean; + peakCount?: number; +} + +export function useTimelineAudioPeaks( + mediaResource: string | null | undefined, + options: TimelineAudioPeaksOptions = {}, +): AudioPeaksData | null { const [data, setData] = useState(null); - const urlRef = useRef(fileUrl); + const sourceRef = useRef(mediaResource); + const enableSourceSidecarFallback = options.enableSourceSidecarFallback ?? false; + const peakCount = options.peakCount ?? WAVEFORM_DEFAULT_PEAK_COUNT; useEffect(() => { - urlRef.current = fileUrl; + sourceRef.current = mediaResource; setData(null); - - if (!fileUrl) { - return; - } + if (!mediaResource) return; let cancelled = false; - (async () => { + const run = async () => { + const tryGenerate = async (resource: string): Promise => { + const resolvedUrl = await resolveMediaResourceUrl(resource); + return waveformGenerator.generate(resolvedUrl, peakCount); + }; + try { - const response = await fetch(fileUrl); - if (cancelled) return; - - const arrayBuffer = await response.arrayBuffer(); - if (cancelled) return; - - const audioCtx = new OfflineAudioContext(1, 1, 44100); - const decoded = await audioCtx.decodeAudioData(arrayBuffer); - if (cancelled) return; - - const channelData = decoded.getChannelData(0); - const durationMs = decoded.duration * 1000; - const binSize = Math.max(1, Math.floor(channelData.length / TARGET_PEAK_COUNT)); - const peakCount = Math.ceil(channelData.length / binSize); - const peaks = new Float32Array(peakCount); - - for (let i = 0; i < peakCount; i++) { - const start = i * binSize; - const end = Math.min(start + binSize, channelData.length); - let max = 0; - for (let j = start; j < end; j++) { - const abs = Math.abs(channelData[j]); - if (abs > max) max = abs; - } - peaks[i] = max; - } - - // Normalise to 0–1 range. - let globalMax = 0; - for (let i = 0; i < peaks.length; i++) { - if (peaks[i] > globalMax) globalMax = peaks[i]; - } - if (globalMax > 0) { - for (let i = 0; i < peaks.length; i++) { - peaks[i] /= globalMax; - } - } - - if (!cancelled && urlRef.current === fileUrl) { - setData({ peaks, durationMs }); - } + const result = await tryGenerate(mediaResource); + if (!cancelled && sourceRef.current === mediaResource) setData(result); + return; } catch { - // File has no audio or decoding failed — leave as null. + // fallthrough } - })(); + + if (!enableSourceSidecarFallback) return; + + const localPathFromServer = extractLocalPathFromMediaServerUrl(mediaResource); + const localSourcePath = + localPathFromServer || + (/^file:\/\//i.test(mediaResource) ? fromFileUrl(mediaResource) : mediaResource); + if (!localSourcePath) return; + + for (const candidate of buildSidecarAudioCandidates(localSourcePath)) { + try { + const result = await tryGenerate(candidate); + if (!cancelled && sourceRef.current === mediaResource) setData(result); + return; + } catch { + // try next + } + } + }; + + void run(); return () => { cancelled = true; }; - }, [fileUrl]); + }, [mediaResource, enableSourceSidecarFallback, peakCount]); return data; } diff --git a/src/components/video-editor/timeline/model/timelineModel.ts b/src/components/video-editor/timeline/model/timelineModel.ts index 0981b85c..96681351 100644 --- a/src/components/video-editor/timeline/model/timelineModel.ts +++ b/src/components/video-editor/timeline/model/timelineModel.ts @@ -47,13 +47,22 @@ export function buildTimelineItems(params: { variant: "zoom", })); - const clips: TimelineRenderItem[] = clipRegions.map((region, index) => ({ - id: region.id, - rowId: CLIP_ROW_ID, - span: { start: region.startMs, end: region.endMs }, - label: `Clip ${index + 1}`, - variant: "clip", - })); + const clips: TimelineRenderItem[] = clipRegions.map((region, index) => { + const displayDurationMs = Math.max(0, region.endMs - region.startMs); + const speed = Number.isFinite(region.speed) && region.speed > 0 ? region.speed : 1; + const sourceEndMs = region.startMs + displayDurationMs * speed; + + return { + id: region.id, + rowId: CLIP_ROW_ID, + span: { start: region.startMs, end: region.endMs }, + sourceSpan: { start: region.startMs, end: sourceEndMs }, + label: `Clip ${index + 1}`, + showSourceAudio: region.showSourceAudio, + muted: Boolean(region.muted), + variant: "clip", + }; + }); const annotations: TimelineRenderItem[] = annotationRegions.map((region) => ({ id: region.id, @@ -68,6 +77,9 @@ export function buildTimelineItems(params: { rowId: getAudioTrackRowId(region.trackIndex ?? 0), span: { start: region.startMs, end: region.endMs }, label: getAudioLabel(region), + audioPath: region.audioPath, + audioGain: region.volume, + audioNormalize: Boolean(region.normalize), variant: "audio", })); diff --git a/src/components/video-editor/types.ts b/src/components/video-editor/types.ts index 92f84fd6..e6a7c6f0 100644 --- a/src/components/video-editor/types.ts +++ b/src/components/video-editor/types.ts @@ -64,6 +64,7 @@ export type EditorEffectSection = | "crop" | "extensions" | "clip" + | "audio" | `ext:${string}`; export type ZoomTransitionEasing = "recordly" | "glide" | "smooth" | "snappy" | "linear"; @@ -169,6 +170,7 @@ export interface ClipRegion { endMs: number; speed: number; muted?: boolean; + showSourceAudio?: boolean; } export function getClipSourceEndMs(clip: ClipRegion): number { @@ -464,6 +466,7 @@ export const DEFAULT_PADDING: Padding = { right: 20, linked: true, }; +export type { SourceAudioTrackSetting, SourceAudioTrackSettings } from "@/components/video-editor/audio/audioTypes"; export interface AudioRegion { id: string; @@ -471,6 +474,7 @@ export interface AudioRegion { endMs: number; audioPath: string; volume: number; + normalize?: boolean; trackIndex?: number; } diff --git a/src/i18n/locales/en/settings.json b/src/i18n/locales/en/settings.json index 3a2b6a76..b9a2d00a 100644 --- a/src/i18n/locales/en/settings.json +++ b/src/i18n/locales/en/settings.json @@ -19,7 +19,10 @@ }, "clip": { "title": "Clip", - "muteAudio": "Mute Audio", + "mute": "Mute", + "mutedState": "Audio is muted", + "unmutedState": "Audio is playing", + "separateClipFromAudio": "Separate clip from audio", "delete": "Delete Clip" }, "effects": { @@ -201,5 +204,16 @@ "exportVideo": "Export {{format}}", "reportBug": "Report Bug", "starOnGithub": "Star on GitHub" + }, + "audio": { + "title": "Audio", + "volumeTitle": "Audio", + "volume": "Volume", + "normalize": "Normalize", + "sourceTracksTitle": "Clip Source Audio", + "systemLabel": "Source System", + "micLabel": "Source Mic", + "mixedLabel": "Source", + "deleteRegion": "Delete Audio" } } diff --git a/src/i18n/locales/es/settings.json b/src/i18n/locales/es/settings.json index 0c7ca244..66a05e89 100644 --- a/src/i18n/locales/es/settings.json +++ b/src/i18n/locales/es/settings.json @@ -19,7 +19,10 @@ }, "clip": { "title": "Clip", - "muteAudio": "Silenciar audio", + "mute": "Silenciar", + "mutedState": "El audio está silenciado", + "unmutedState": "El audio se está reproduciendo", + "separateClipFromAudio": "Separar clip del audio", "delete": "Eliminar clip" }, "effects": { @@ -181,5 +184,16 @@ "exportVideo": "Exportar {{format}}", "reportBug": "Reportar error", "starOnGithub": "Estrella en GitHub" + }, + "audio": { + "title": "Audio", + "volumeTitle": "Audio", + "volume": "Volumen", + "normalize": "Normalizar", + "sourceTracksTitle": "Audio fuente del clip", + "systemLabel": "Sonido del sistema", + "micLabel": "Micrófono", + "mixedLabel": "Fuente", + "deleteRegion": "Eliminar audio" } } diff --git a/src/i18n/locales/fr/settings.json b/src/i18n/locales/fr/settings.json index 04804828..200ecaa0 100644 --- a/src/i18n/locales/fr/settings.json +++ b/src/i18n/locales/fr/settings.json @@ -19,7 +19,10 @@ }, "clip": { "title": "Clip", - "muteAudio": "Couper le son", + "mute": "Sourdine", + "mutedState": "Le son est coupé", + "unmutedState": "Le son est activé", + "separateClipFromAudio": "Séparer le clip de l'audio", "delete": "Supprimer le clip" }, "effects": { @@ -181,5 +184,16 @@ "exportVideo": "Exporter en {{format}}", "reportBug": "Signaler un bug", "starOnGithub": "Mettre une étoile sur GitHub" + }, + "audio": { + "title": "Audio", + "volumeTitle": "Audio", + "volume": "Volume", + "normalize": "Normaliser", + "sourceTracksTitle": "Source audio du clip", + "systemLabel": "Son système", + "micLabel": "Microphone", + "mixedLabel": "Source", + "deleteRegion": "Supprimer la zone audio" } } diff --git a/src/i18n/locales/ko/settings.json b/src/i18n/locales/ko/settings.json index 8c32549a..c528a98f 100644 --- a/src/i18n/locales/ko/settings.json +++ b/src/i18n/locales/ko/settings.json @@ -19,7 +19,10 @@ }, "clip": { "title": "클립", - "muteAudio": "오디오 음소거", + "mute": "음소거", + "mutedState": "오디오가 음소거됨", + "unmutedState": "오디오가 재생 중", + "separateClipFromAudio": "클립에서 오디오 분리", "delete": "클립 삭제" }, "effects": { @@ -181,5 +184,16 @@ "exportVideo": "{{format}} 내보내기", "reportBug": "버그 신고", "starOnGithub": "GitHub에서 별표 주기" + }, + "audio": { + "title": "오디오", + "volumeTitle": "오디오", + "volume": "볼륨", + "normalize": "정규화", + "sourceTracksTitle": "클립 소스 오디오", + "systemLabel": "시스템 소스", + "micLabel": "마이크 소스", + "mixedLabel": "소스", + "deleteRegion": "오디오 삭제" } } diff --git a/src/i18n/locales/nl/settings.json b/src/i18n/locales/nl/settings.json index 1e7bb77d..d88e4278 100644 --- a/src/i18n/locales/nl/settings.json +++ b/src/i18n/locales/nl/settings.json @@ -19,7 +19,10 @@ }, "clip": { "title": "Clip", - "muteAudio": "Audio dempen", + "mute": "Dempen", + "mutedState": "Audio is gedempt", + "unmutedState": "Audio wordt afgespeeld", + "separateClipFromAudio": "Clip van audio scheiden", "delete": "Clip verwijderen" }, "effects": { @@ -181,5 +184,16 @@ "exportVideo": "{{format}} exporteren", "reportBug": "Bug melden", "starOnGithub": "Ster op GitHub" + }, + "audio": { + "title": "Audio", + "volumeTitle": "Audio", + "volume": "Volume", + "normalize": "Normaliseren", + "sourceTracksTitle": "Bronaudio clip", + "systemLabel": "Systeemaudio", + "micLabel": "Microfoon", + "mixedLabel": "Bron", + "deleteRegion": "Audio verwijderen" } } diff --git a/src/i18n/locales/pt-BR/settings.json b/src/i18n/locales/pt-BR/settings.json index 58783df0..a34e6e4d 100644 --- a/src/i18n/locales/pt-BR/settings.json +++ b/src/i18n/locales/pt-BR/settings.json @@ -19,7 +19,10 @@ }, "clip": { "title": "Clipe", - "muteAudio": "Silenciar áudio", + "mute": "Mudo", + "mutedState": "O áudio está mudo", + "unmutedState": "O áudio está tocando", + "separateClipFromAudio": "Separar áudio do clipe", "delete": "Excluir clipe" }, "effects": { @@ -181,5 +184,16 @@ "exportVideo": "Exportar {{format}}", "reportBug": "Reportar bug", "starOnGithub": "Dar estrela no GitHub" + }, + "audio": { + "title": "Áudio", + "volumeTitle": "Áudio", + "volume": "Volume", + "normalize": "Normalizar", + "sourceTracksTitle": "Áudio original do clipe", + "systemLabel": "Sistema", + "micLabel": "Microfone", + "mixedLabel": "Fonte", + "deleteRegion": "Excluir áudio" } } diff --git a/src/i18n/locales/ru/settings.json b/src/i18n/locales/ru/settings.json index 88e48ec5..687f8383 100644 --- a/src/i18n/locales/ru/settings.json +++ b/src/i18n/locales/ru/settings.json @@ -19,8 +19,11 @@ }, "clip": { "title": "Клип", - "muteAudio": "Выключить звук", - "delete": "Удалить" + "mute": "Без звука", + "mutedState": "Звук выключен", + "unmutedState": "Звук включен", + "separateClipFromAudio": "Отделить аудио от клипа", + "delete": "Удалить клип" }, "effects": { "title": "Эффекты", @@ -201,5 +204,16 @@ "exportVideo": "Экспортировать {{format}}", "reportBug": "Сообщить об ошибке", "starOnGithub": "Оценить на GitHub" + }, + "audio": { + "title": "Аудио", + "volumeTitle": "Аудио", + "volume": "Громкость", + "normalize": "Нормализовать", + "sourceTracksTitle": "Исходное аудио клипа", + "systemLabel": "Системный звук", + "micLabel": "Микрофон", + "mixedLabel": "Источник", + "deleteRegion": "Удалить аудио" } } \ No newline at end of file diff --git a/src/i18n/locales/zh-CN/settings.json b/src/i18n/locales/zh-CN/settings.json index 5f5b277a..a68d2a04 100644 --- a/src/i18n/locales/zh-CN/settings.json +++ b/src/i18n/locales/zh-CN/settings.json @@ -19,7 +19,10 @@ }, "clip": { "title": "片段", - "muteAudio": "静音音频", + "mute": "静音", + "mutedState": "音频已静音", + "unmutedState": "音频正在播放", + "separateClipFromAudio": "将剪辑与音频分离", "delete": "删除片段" }, "effects": { @@ -195,6 +198,17 @@ "saveProject": "保存项目", "exportVideo": "导出{{format}}", "reportBug": "报告问题", - "starOnGithub": "在 GitHub 上加星" + "starOnGithub": "在 GitHub 上点赞" + }, + "audio": { + "title": "音频", + "volumeTitle": "音频", + "volume": "音量", + "normalize": "标准化", + "sourceTracksTitle": "剪辑源音频", + "systemLabel": "系统声音", + "micLabel": "麦克风", + "mixedLabel": "来源", + "deleteRegion": "删除音频" } } diff --git a/src/i18n/locales/zh-TW/settings.json b/src/i18n/locales/zh-TW/settings.json index bf2ffc9c..db0ea7e0 100644 --- a/src/i18n/locales/zh-TW/settings.json +++ b/src/i18n/locales/zh-TW/settings.json @@ -19,7 +19,10 @@ }, "clip": { "title": "片段", - "muteAudio": "靜音", + "mute": "靜音", + "mutedState": "音訊已靜音", + "unmutedState": "音訊正在播放", + "separateClipFromAudio": "將片段與音訊分離", "delete": "刪除片段" }, "effects": { @@ -181,5 +184,16 @@ "exportVideo": "匯出 {{format}}", "reportBug": "回報錯誤", "starOnGithub": "在 GitHub 按讚" + }, + "audio": { + "title": "音訊", + "volumeTitle": "音訊", + "volume": "音量", + "normalize": "正規化", + "sourceTracksTitle": "剪輯源音訊", + "systemLabel": "系統源", + "micLabel": "麥克風源", + "mixedLabel": "源", + "deleteRegion": "刪除音訊" } } diff --git a/src/lib/exporter/audioEncoder.ts b/src/lib/exporter/audioEncoder.ts index 647c9768..8e98091e 100644 --- a/src/lib/exporter/audioEncoder.ts +++ b/src/lib/exporter/audioEncoder.ts @@ -3,12 +3,18 @@ import type { AudioRegion, ClipRegion, SpeedRegion, + SourceAudioTrackSettings, TrimRegion, } from "@/components/video-editor/types"; +import { + buildResolvedAudioPlan, + SourceTrackId, +} from "@/lib/exporter/audioRoutingEngine"; import { estimateCompanionAudioStartDelaySeconds } from "@/lib/mediaTiming"; import { resolveMediaElementSource } from "./localMediaSource"; import type { VideoMuxer } from "./muxer"; -import { resolveSourceAudioFallbackPaths } from "./sourceAudioFallback"; +import { resolveSourceTrackRoutingPolicy } from "./sourceTrackRoutingPolicy"; +import { SOURCE_AUDIO_NORMALIZE_GAIN } from "@/components/video-editor/audio/audioTypes"; const AUDIO_BITRATE = 128_000; const DECODE_BACKPRESSURE_LIMIT = 20; @@ -19,6 +25,60 @@ const OFFLINE_AUDIO_SAMPLE_RATE = 48_000; const OFFLINE_ENCODE_CHUNK_FRAMES = 1024; const OFFLINE_CHUNK_DURATION_SEC = 30; +function resolveSourceTrackGain( + sourceAudioTrackSettings: SourceAudioTrackSettings | undefined, + trackId: "mic" | "system" | "mixed", +) { + const settings = sourceAudioTrackSettings?.[trackId]; + if (!settings) { + return 1; + } + const normalizeGain = settings.normalize ? SOURCE_AUDIO_NORMALIZE_GAIN : 1; + return Math.max(0, Math.min(2, settings.volume * normalizeGain)); +} + +export function getSourceTrackIdFromPath(audioPath: string): SourceTrackId { + const normalized = audioPath.toLowerCase(); + // Check for common patterns like .mic., -mic., mic.mp4, etc. + if ( + normalized.includes(".mic.") || + normalized.includes("-mic.") || + normalized.includes("_mic_") || + normalized.includes("/mic.") || + normalized.includes("\\mic.") || + normalized.endsWith("mic.mp4") || + normalized.endsWith("mic.m4a") || + normalized.endsWith("mic.wav") + ) { + return "mic"; + } + if ( + normalized.includes(".system.") || + normalized.includes("-system.") || + normalized.includes("_system_") || + normalized.includes("/system.") || + normalized.includes("\\system.") || + normalized.endsWith("system.mp4") || + normalized.endsWith("system.m4a") || + normalized.endsWith("system.wav") + ) { + return "system"; + } + return "mixed"; +} + +export function hasNonDefaultSourceTrackSettings( + sourceAudioTrackSettings?: SourceAudioTrackSettings, +) { + if (!sourceAudioTrackSettings) { + return false; + } + return Object.values(sourceAudioTrackSettings).some( + (settings) => + Math.abs((settings?.volume ?? 1) - 1) > 0.0005 || Boolean(settings?.normalize), + ); +} + interface TimelineSlice { sourceStartMs: number; sourceEndMs: number; @@ -26,9 +86,10 @@ interface TimelineSlice { } interface PreparedOfflineRender { - mainBuffer: AudioBuffer | null; - companionEntries: Array<{ buffer: AudioBuffer; startDelaySec: number }>; + mainBufferEntry: { buffer: AudioBuffer; gain: number } | null; + companionEntries: Array<{ buffer: AudioBuffer; startDelaySec: number; gain: number }>; regionEntries: Array<{ buffer: AudioBuffer; region: AudioRegion }>; + mutedSourceOutputRangesSec: Array<{ startSec: number; endSec: number }>; slices: TimelineSlice[]; outputDurationMs: number; numChannels: number; @@ -144,6 +205,8 @@ export class AudioProcessor { audioRegions?: AudioRegion[], sourceAudioFallbackPaths?: string[], sourceAudioFallbackStartDelayMsByPath?: Record, + sourceAudioTrackSettings?: SourceAudioTrackSettings, + clipRegions?: ClipRegion[], ): Promise { const sortedTrims = trimRegions ? [...trimRegions].sort((a, b) => a.startMs - b.startMs) @@ -161,23 +224,25 @@ export class AudioProcessor { (audioPath) => typeof audioPath === "string" && audioPath.trim().length > 0, ) : []; - const { hasEmbeddedSourceAudio, externalAudioPaths } = resolveSourceAudioFallbackPaths( + const routingPolicy = resolveSourceTrackRoutingPolicy( videoUrl, sortedSourceAudioFallbackPaths, ); - const hasTimedCompanionAudio = externalAudioPaths.some( + const hasTimedCompanionAudio = routingPolicy.playbackPaths.some( (audioPath) => (sourceAudioFallbackStartDelayMsByPath?.[audioPath] ?? 0) > 0, ); const needsSourceAudioMixing = - externalAudioPaths.length > 1 || - (hasEmbeddedSourceAudio && externalAudioPaths.length > 0) || + routingPolicy.playbackPaths.length > 1 || + (routingPolicy.hasEmbeddedSourceAudio && routingPolicy.playbackPaths.length > 0) || hasTimedCompanionAudio; // When speed edits, audio regions, or multiple audio sources need mixing, use offline AudioContext pipeline. if ( sortedSpeedRegions.length > 0 || sortedAudioRegions.length > 0 || - needsSourceAudioMixing + needsSourceAudioMixing || + hasNonDefaultSourceTrackSettings(sourceAudioTrackSettings) || + (clipRegions ?? []).some((clip) => Boolean(clip.muted)) ) { await this.renderAndMuxOfflineAudio( videoUrl, @@ -186,14 +251,16 @@ export class AudioProcessor { sortedAudioRegions, sortedSourceAudioFallbackPaths, sourceAudioFallbackStartDelayMsByPath, + sourceAudioTrackSettings, + clipRegions, muxer, ); return; } // Single sidecar audio with no speed/audio edits: demux directly (skips slow real-time rendering). - if (!hasEmbeddedSourceAudio && externalAudioPaths.length === 1) { - const sidecarDemuxer = await this.loadAudioFileDemuxer(externalAudioPaths[0]); + if (!routingPolicy.hasEmbeddedSourceAudio && routingPolicy.playbackPaths.length === 1) { + const sidecarDemuxer = await this.loadAudioFileDemuxer(routingPolicy.playbackPaths[0]); if (sidecarDemuxer) { try { await this.processTrimOnlyAudio(sidecarDemuxer, muxer, sortedTrims); @@ -215,8 +282,10 @@ export class AudioProcessor { sortedTrims, [], [], - externalAudioPaths, + routingPolicy.playbackPaths, sourceAudioFallbackStartDelayMsByPath, + sourceAudioTrackSettings, + clipRegions, muxer, ); return; @@ -263,6 +332,8 @@ export class AudioProcessor { audioRegions?: AudioRegion[], sourceAudioFallbackPaths?: string[], sourceAudioFallbackStartDelayMsByPath?: Record, + sourceAudioTrackSettings?: SourceAudioTrackSettings, + clipRegions?: ClipRegion[], ): Promise { const sortedTrims = trimRegions ? [...trimRegions].sort((a, b) => a.startMs - b.startMs) @@ -288,6 +359,8 @@ export class AudioProcessor { sortedAudioRegions, sortedSourceAudioFallbackPaths, sourceAudioFallbackStartDelayMsByPath, + sourceAudioTrackSettings, + clipRegions, ); return this.renderToWavBlobChunked(prepared); } @@ -563,6 +636,8 @@ export class AudioProcessor { audioRegions: AudioRegion[], sourceAudioFallbackPaths: string[], sourceAudioFallbackStartDelayMsByPath: Record | undefined, + sourceAudioTrackSettings: SourceAudioTrackSettings | undefined, + clipRegions: ClipRegion[] | undefined, muxer: VideoMuxer, ): Promise { const prepared = await this.prepareOfflineRender( @@ -572,6 +647,8 @@ export class AudioProcessor { audioRegions, sourceAudioFallbackPaths, sourceAudioFallbackStartDelayMsByPath, + sourceAudioTrackSettings, + clipRegions, ); if (this.cancelled) return; await this.renderAndEncodeChunked(prepared, muxer); @@ -584,31 +661,59 @@ export class AudioProcessor { audioRegions: AudioRegion[], sourceAudioFallbackPaths: string[], sourceAudioFallbackStartDelayMsByPath?: Record, + sourceAudioTrackSettings?: SourceAudioTrackSettings, + clipRegions?: ClipRegion[], ): Promise { if (this.cancelled) throw new Error("Export cancelled"); this.onProgress?.(0); - const { externalAudioPaths } = resolveSourceAudioFallbackPaths( - videoUrl, + const resolvedPlan = buildResolvedAudioPlan({ + videoResource: videoUrl, sourceAudioFallbackPaths, - ); + audioRegions, + sourceTrackGainById: { + mic: resolveSourceTrackGain(sourceAudioTrackSettings, "mic"), + system: resolveSourceTrackGain(sourceAudioTrackSettings, "system"), + mixed: resolveSourceTrackGain(sourceAudioTrackSettings, "mixed"), + }, + embeddedGain: Math.max( + 0, + Math.min( + 2, + sourceAudioTrackSettings?.mixed + ? resolveSourceTrackGain(sourceAudioTrackSettings, "mixed") + : sourceAudioTrackSettings?.system + ? resolveSourceTrackGain(sourceAudioTrackSettings, "system") + : 1, + ), + ), + }); // Decode embedded source audio separately from companion sidecars. - const mainBuffer = await this.decodeAudioFromUrl(videoUrl); + const mainBuffer = resolvedPlan.includeEmbeddedInExport + ? await this.decodeAudioFromUrl(videoUrl) + : null; + const mainBufferGain = resolveSourceTrackGain(sourceAudioTrackSettings, "mixed"); + const mainBufferEntry = mainBuffer ? { buffer: mainBuffer, gain: mainBufferGain } : null; if (this.cancelled) throw new Error("Export cancelled"); // Decode companion / sidecar audio files - const companionEntries: Array<{ buffer: AudioBuffer; startDelaySec: number }> = []; + const companionEntries: Array<{ buffer: AudioBuffer; startDelaySec: number; gain: number }> = + []; const refDuration = mainBuffer?.duration ?? - (externalAudioPaths.length > 0 ? await this.getMediaDurationSec(videoUrl) : 0); - for (const audioPath of externalAudioPaths) { + (resolvedPlan.playbackPaths.length > 0 ? await this.getMediaDurationSec(videoUrl) : 0); + for (const audioPath of resolvedPlan.playbackPaths) { if (this.cancelled) throw new Error("Export cancelled"); const buffer = await this.decodeAudioFromUrl(audioPath); if (!buffer) continue; companionEntries.push({ buffer, + gain: resolveSourceTrackGain( + sourceAudioTrackSettings, + getSourceTrackIdFromPath(audioPath), + ), startDelaySec: estimateCompanionAudioStartDelaySeconds( refDuration, buffer.duration, @@ -629,15 +734,15 @@ export class AudioProcessor { this.onProgress?.(0.2); // Determine source duration for timeline calculation - const primaryBuffer = mainBuffer ?? companionEntries[0]?.buffer ?? null; + const primaryBuffer = mainBufferEntry?.buffer ?? companionEntries[0]?.buffer ?? null; if (!primaryBuffer && regionEntries.length === 0) { throw new Error("No decodable audio sources found"); } let sourceDurationSec: number; - if (mainBuffer) { - sourceDurationSec = mainBuffer.duration; - } else if (externalAudioPaths.length > 0 || regionEntries.length > 0) { + if (mainBufferEntry?.buffer) { + sourceDurationSec = mainBufferEntry.buffer.duration; + } else if (resolvedPlan.playbackPaths.length > 0 || regionEntries.length > 0) { sourceDurationSec = await this.getMediaDurationSec(videoUrl); } else { sourceDurationSec = primaryBuffer?.duration ?? 0; @@ -659,11 +764,24 @@ export class AudioProcessor { } const numChannels = Math.min(primaryBuffer?.numberOfChannels ?? 2, 2); + const mutedSourceOutputRangesSec = (clipRegions ?? []) + .filter( + (clip) => + Boolean(clip.muted) && + Number.isFinite(clip.startMs) && + Number.isFinite(clip.endMs) && + clip.endMs > clip.startMs, + ) + .map((clip) => ({ + startSec: Math.max(0, clip.startMs / 1000), + endSec: Math.max(0, clip.endMs / 1000), + })); return { - mainBuffer, + mainBufferEntry, companionEntries, regionEntries, + mutedSourceOutputRangesSec, slices, outputDurationMs, numChannels, @@ -788,14 +906,16 @@ export class AudioProcessor { ); // Schedule main audio - if (prepared.mainBuffer) { + if (prepared.mainBufferEntry) { this.scheduleBufferThroughTimeline( offlineCtx, - prepared.mainBuffer, + prepared.mainBufferEntry.buffer, slices, 0, + prepared.mainBufferEntry.gain, outputOffsetSec, chunkSec, + prepared.mutedSourceOutputRangesSec, ); } @@ -806,8 +926,10 @@ export class AudioProcessor { entry.buffer, slices, entry.startDelaySec, + entry.gain, outputOffsetSec, chunkSec, + prepared.mutedSourceOutputRangesSec, ); } @@ -865,7 +987,8 @@ export class AudioProcessor { if (duration <= 0.001) return; const gainNode = ctx.createGain(); - gainNode.gain.value = Math.max(0, Math.min(1, region.volume)); + const normalizeGain = region.normalize ? SOURCE_AUDIO_NORMALIZE_GAIN : 1; + gainNode.gain.value = Math.max(0, Math.min(1, region.volume * normalizeGain)); gainNode.connect(ctx.destination); const source = ctx.createBufferSource(); @@ -1265,8 +1388,10 @@ export class AudioProcessor { buffer: AudioBuffer, slices: TimelineSlice[], sourceStartDelaySec: number, + gain = 1, chunkOutputStartSec = 0, chunkDurationSec = Number.POSITIVE_INFINITY, + mutedOutputRangesSec: Array<{ startSec: number; endSec: number }> = [], ): void { let outputOffsetSec = 0; @@ -1330,12 +1455,51 @@ export class AudioProcessor { continue; } - const source = ctx.createBufferSource(); - source.buffer = buffer; - source.playbackRate.value = slice.speed; - source.connect(ctx.destination); + const audibleRanges: Array<{ startSec: number; endSec: number }> = [ + { + startSec: localOutputStartSec + chunkOutputStartSec, + endSec: + localOutputStartSec + chunkOutputStartSec + effectiveSourceDurationSec / slice.speed, + }, + ]; + for (const mutedRange of mutedOutputRangesSec) { + for (let rangeIndex = audibleRanges.length - 1; rangeIndex >= 0; rangeIndex -= 1) { + const current = audibleRanges[rangeIndex]; + const overlapStart = Math.max(current.startSec, mutedRange.startSec); + const overlapEnd = Math.min(current.endSec, mutedRange.endSec); + if (overlapEnd <= overlapStart) { + continue; + } + audibleRanges.splice(rangeIndex, 1); + if (current.startSec < overlapStart) { + audibleRanges.push({ startSec: current.startSec, endSec: overlapStart }); + } + if (overlapEnd < current.endSec) { + audibleRanges.push({ startSec: overlapEnd, endSec: current.endSec }); + } + } + } - source.start(localOutputStartSec, effectiveBufferStartSec, effectiveSourceDurationSec); + for (const audibleRange of audibleRanges) { + const audibleDurationSec = audibleRange.endSec - audibleRange.startSec; + if (audibleDurationSec <= 0.001) { + continue; + } + const source = ctx.createBufferSource(); + const gainNode = ctx.createGain(); + gainNode.gain.value = Math.max(0, Math.min(2, gain)); + source.buffer = buffer; + source.playbackRate.value = slice.speed; + source.connect(gainNode); + gainNode.connect(ctx.destination); + + const sourceOffsetSec = + effectiveBufferStartSec + + (audibleRange.startSec - (localOutputStartSec + chunkOutputStartSec)) * slice.speed; + const localStartSec = audibleRange.startSec - chunkOutputStartSec; + const sourceDurationSec = audibleDurationSec * slice.speed; + source.start(localStartSec, sourceOffsetSec, sourceDurationSec); + } outputOffsetSec += sliceOutputDurationSec; } diff --git a/src/lib/exporter/audioRoutingEngine.ts b/src/lib/exporter/audioRoutingEngine.ts new file mode 100644 index 00000000..3fc3285f --- /dev/null +++ b/src/lib/exporter/audioRoutingEngine.ts @@ -0,0 +1,129 @@ +import type { AudioRegion } from "@/components/video-editor/types"; +import { SOURCE_AUDIO_NORMALIZE_GAIN } from "@/components/video-editor/audio/audioTypes"; +import { resolveSourceAudioFallbackPaths } from "./sourceAudioFallback"; + +export type SourceTrackId = "mic" | "system" | "mixed"; +export type ResolvedAudioTrackKind = "user" | "system" | "mic" | "mixed" | "embedded"; + +export interface ResolvedAudioTrack { + id: string; + kind: ResolvedAudioTrackKind; + sourceRef: { + path: string; + startDelayMs: number; + }; + gain: number; + timelineBinding: { + startMs: number; + endMs: number; + }; +} + +export interface ResolvedAudioPlan { + hasEmbeddedSourceAudio: boolean; + pathsByTrack: Partial>; + playbackPaths: string[]; + muteEmbeddedPreview: boolean; + includeEmbeddedInExport: boolean; + tracks: ResolvedAudioTrack[]; + masterGain: number; +} + +export function getSourceTrackIdFromPath(audioPath: string): SourceTrackId { + const normalized = audioPath.toLowerCase(); + if (normalized.includes(".mic.")) return "mic"; + if (normalized.includes(".system.")) return "system"; + return "mixed"; +} + +function clampGain(value: number, max: number) { + if (!Number.isFinite(value)) return 1; + return Math.max(0, Math.min(max, value)); +} + +export function buildResolvedAudioPlan(input: { + videoResource: string | null | undefined; + sourceAudioFallbackPaths: string[] | null | undefined; + audioRegions?: AudioRegion[]; + sourceTrackGainById?: Partial>; + embeddedGain?: number; + masterGain?: number; +}): ResolvedAudioPlan { + const { hasEmbeddedSourceAudio, externalAudioPaths } = resolveSourceAudioFallbackPaths( + input.videoResource, + input.sourceAudioFallbackPaths, + ); + + const pathsByTrack: Partial> = {}; + for (const path of externalAudioPaths) { + const trackId = getSourceTrackIdFromPath(path); + if (!pathsByTrack[trackId]) { + pathsByTrack[trackId] = path; + } + } + + const hasDedicatedTracks = Boolean(pathsByTrack.system || pathsByTrack.mic); + const playbackPaths: string[] = []; + if (pathsByTrack.system) playbackPaths.push(pathsByTrack.system); + if (pathsByTrack.mic) playbackPaths.push(pathsByTrack.mic); + if (!hasDedicatedTracks && pathsByTrack.mixed) playbackPaths.push(pathsByTrack.mixed); + + const includeEmbeddedInExport = !pathsByTrack.system && !pathsByTrack.mixed; + const resolvedRegions = (input.audioRegions ?? []).slice().sort((a, b) => a.startMs - b.startMs); + const tracks: ResolvedAudioTrack[] = resolvedRegions.map((region) => ({ + id: `user:${region.id}`, + kind: "user", + sourceRef: { + path: region.audioPath, + startDelayMs: 0, + }, + gain: clampGain(region.volume * (region.normalize ? SOURCE_AUDIO_NORMALIZE_GAIN : 1), 1), + timelineBinding: { + startMs: Math.max(0, region.startMs), + endMs: Math.max(0, region.endMs), + }, + })); + + for (const audioPath of playbackPaths) { + const trackId = getSourceTrackIdFromPath(audioPath); + tracks.push({ + id: `${trackId}:${audioPath}`, + kind: trackId, + sourceRef: { + path: audioPath, + startDelayMs: 0, + }, + gain: clampGain(input.sourceTrackGainById?.[trackId] ?? 1, 2), + timelineBinding: { + startMs: 0, + endMs: Number.POSITIVE_INFINITY, + }, + }); + } + + if (hasEmbeddedSourceAudio && input.videoResource) { + tracks.push({ + id: `embedded:${input.videoResource}`, + kind: "embedded", + sourceRef: { + path: input.videoResource, + startDelayMs: 0, + }, + gain: clampGain(input.embeddedGain ?? input.sourceTrackGainById?.mixed ?? 1, 2), + timelineBinding: { + startMs: 0, + endMs: Number.POSITIVE_INFINITY, + }, + }); + } + + return { + hasEmbeddedSourceAudio, + pathsByTrack, + playbackPaths, + muteEmbeddedPreview: hasDedicatedTracks && !includeEmbeddedInExport, + includeEmbeddedInExport, + tracks, + masterGain: clampGain(input.masterGain ?? 1, 1), + }; +} diff --git a/src/lib/exporter/modernVideoExporter.ts b/src/lib/exporter/modernVideoExporter.ts index ce8b3eae..55f47923 100644 --- a/src/lib/exporter/modernVideoExporter.ts +++ b/src/lib/exporter/modernVideoExporter.ts @@ -3,11 +3,13 @@ import type { AudioRegion, AutoCaptionSettings, CaptionCue, + ClipRegion, CropRegion, CursorStyle, CursorTelemetryPoint, Padding, SpeedRegion, + SourceAudioTrackSettings, TrimRegion, WebcamOverlaySettings, ZoomMotionBlurTuning, @@ -135,8 +137,10 @@ interface VideoExporterConfig extends ExportConfig { zoomClassicMode?: boolean; frame?: string | null; audioRegions?: AudioRegion[]; + clipRegions?: ClipRegion[]; sourceAudioFallbackPaths?: string[]; sourceAudioFallbackStartDelayMsByPath?: Record; + sourceAudioTrackSettings?: SourceAudioTrackSettings; previewWidth?: number; previewHeight?: number; onProgress?: (progress: ExportProgress) => void; @@ -167,6 +171,16 @@ type NativeAudioPlan = }; const FILTERGRAPH_FALLBACK_AUDIO_SAMPLE_RATE = 48_000; + +function hasNonDefaultSourceTrackSettings(sourceAudioTrackSettings?: SourceAudioTrackSettings) { + if (!sourceAudioTrackSettings) { + return false; + } + return Object.values(sourceAudioTrackSettings).some( + (settings) => + Math.abs((settings?.volume ?? 1) - 1) > 0.0005 || Boolean(settings?.normalize), + ); +} const MIN_NATIVE_STATIC_LAYOUT_SPEED = 0.25; const MAX_NATIVE_STATIC_LAYOUT_SPEED = 30; @@ -752,6 +766,8 @@ export class ModernVideoExporter { this.config.audioRegions, this.config.sourceAudioFallbackPaths, this.config.sourceAudioFallbackStartDelayMsByPath, + this.config.sourceAudioTrackSettings, + this.config.clipRegions, ), "audio processing", "audio", @@ -1181,7 +1197,9 @@ export class ModernVideoExporter { speedRegions.length > 0 || audioRegions.length > 0 || sourceAudioFallbackPaths.length > 1 || - hasTimedSourceAudioFallback + hasTimedSourceAudioFallback || + hasNonDefaultSourceTrackSettings(this.config.sourceAudioTrackSettings) || + (this.config.clipRegions ?? []).some((clip) => Boolean(clip.muted)) ) { const sourceDurationMs = Math.max( 0, @@ -1201,16 +1219,20 @@ export class ModernVideoExporter { typeof primaryAudioSourceSampleRate === "number" && Number.isFinite(primaryAudioSourceSampleRate) && primaryAudioSourceSampleRate > 0; - const strategy = canUsePrimaryAudioFiltergraph - ? classifyEditedTrackStrategy({ - primaryAudioSourcePath, - sourceDurationMs, - trimRegions, - speedRegions, - audioRegions, - sourceAudioFallbackPaths, - }) - : "offline-render-fallback"; + const requiresRenderedEditedTrack = + hasNonDefaultSourceTrackSettings(this.config.sourceAudioTrackSettings) || + (this.config.clipRegions ?? []).some((clip) => Boolean(clip.muted)); + const strategy = + canUsePrimaryAudioFiltergraph && !requiresRenderedEditedTrack + ? classifyEditedTrackStrategy({ + primaryAudioSourcePath, + sourceDurationMs, + trimRegions, + speedRegions, + audioRegions, + sourceAudioFallbackPaths, + }) + : "offline-render-fallback"; if (strategy === "filtergraph-fast-path") { const audioSourcePath = primaryAudioSourcePath; @@ -1805,6 +1827,8 @@ export class ModernVideoExporter { this.config.audioRegions, this.config.sourceAudioFallbackPaths, this.config.sourceAudioFallbackStartDelayMsByPath, + this.config.sourceAudioTrackSettings, + this.config.clipRegions, ), description, "audio", diff --git a/src/lib/exporter/sourceTrackRoutingPolicy.test.ts b/src/lib/exporter/sourceTrackRoutingPolicy.test.ts new file mode 100644 index 00000000..58743acc --- /dev/null +++ b/src/lib/exporter/sourceTrackRoutingPolicy.test.ts @@ -0,0 +1,41 @@ +import { describe, expect, it } from "vitest"; +import { resolveSourceTrackRoutingPolicy } from "./sourceTrackRoutingPolicy"; + +describe("resolveSourceTrackRoutingPolicy", () => { + it("prioritizes system+mic sidecars and mutes embedded preview", () => { + const policy = resolveSourceTrackRoutingPolicy("/tmp/recording.mp4", [ + "/tmp/recording.mp4", + "/tmp/recording.system.wav", + "/tmp/recording.mic.wav", + "/tmp/recording.mixed.wav", + ]); + + expect(policy.playbackPaths).toEqual([ + "/tmp/recording.system.wav", + "/tmp/recording.mic.wav", + ]); + expect(policy.muteEmbeddedPreview).toBe(true); + expect(policy.includeEmbeddedInExport).toBe(false); + }); + + it("falls back to mixed when dedicated tracks are absent", () => { + const policy = resolveSourceTrackRoutingPolicy("/tmp/recording.mp4", [ + "/tmp/recording.mixed.wav", + ]); + + expect(policy.playbackPaths).toEqual(["/tmp/recording.mixed.wav"]); + expect(policy.muteEmbeddedPreview).toBe(false); + expect(policy.includeEmbeddedInExport).toBe(false); + }); + + it("keeps embedded audio when only mic sidecar is present", () => { + const policy = resolveSourceTrackRoutingPolicy("/tmp/recording.mp4", [ + "/tmp/recording.mp4", + "/tmp/recording.mic.wav", + ]); + + expect(policy.playbackPaths).toEqual(["/tmp/recording.mic.wav"]); + expect(policy.muteEmbeddedPreview).toBe(false); + expect(policy.includeEmbeddedInExport).toBe(true); + }); +}); diff --git a/src/lib/exporter/sourceTrackRoutingPolicy.ts b/src/lib/exporter/sourceTrackRoutingPolicy.ts new file mode 100644 index 00000000..05c9fa84 --- /dev/null +++ b/src/lib/exporter/sourceTrackRoutingPolicy.ts @@ -0,0 +1,30 @@ +import { + buildResolvedAudioPlan, + type SourceTrackId, +} from "./audioRoutingEngine"; + +export interface SourceTrackRoutingPolicy { + hasEmbeddedSourceAudio: boolean; + pathsByTrack: Partial>; + playbackPaths: string[]; + muteEmbeddedPreview: boolean; + includeEmbeddedInExport: boolean; +} + +export function resolveSourceTrackRoutingPolicy( + videoResource: string | null | undefined, + sourceAudioFallbackPaths: string[] | null | undefined, +): SourceTrackRoutingPolicy { + const plan = buildResolvedAudioPlan({ + videoResource, + sourceAudioFallbackPaths, + }); + + return { + hasEmbeddedSourceAudio: plan.hasEmbeddedSourceAudio, + pathsByTrack: plan.pathsByTrack, + playbackPaths: plan.playbackPaths, + muteEmbeddedPreview: plan.muteEmbeddedPreview, + includeEmbeddedInExport: plan.includeEmbeddedInExport, + }; +} diff --git a/src/lib/exporter/videoExporter.ts b/src/lib/exporter/videoExporter.ts index cf8f6337..b99f5f96 100644 --- a/src/lib/exporter/videoExporter.ts +++ b/src/lib/exporter/videoExporter.ts @@ -3,11 +3,13 @@ import type { AudioRegion, AutoCaptionSettings, CaptionCue, + ClipRegion, CropRegion, CursorStyle, CursorTelemetryPoint, Padding, SpeedRegion, + SourceAudioTrackSettings, TrimRegion, WebcamOverlaySettings, ZoomMotionBlurTuning, @@ -90,8 +92,10 @@ interface VideoExporterConfig extends ExportConfig { zoomSmoothness?: number; frame?: string | null; audioRegions?: AudioRegion[]; + clipRegions?: ClipRegion[]; sourceAudioFallbackPaths?: string[]; sourceAudioFallbackStartDelayMsByPath?: Record; + sourceAudioTrackSettings?: SourceAudioTrackSettings; previewWidth?: number; previewHeight?: number; onProgress?: (progress: ExportProgress) => void; @@ -121,6 +125,16 @@ type NativeAudioPlan = const FILTERGRAPH_FALLBACK_AUDIO_SAMPLE_RATE = 48_000; +function hasNonDefaultSourceTrackSettings(sourceAudioTrackSettings?: SourceAudioTrackSettings) { + if (!sourceAudioTrackSettings) { + return false; + } + return Object.values(sourceAudioTrackSettings).some( + (settings) => + Math.abs((settings?.volume ?? 1) - 1) > 0.0005 || Boolean(settings?.normalize), + ); +} + export class VideoExporter { private config: VideoExporterConfig; private streamingDecoder: StreamingVideoDecoder | null = null; @@ -398,6 +412,7 @@ export class VideoExporter { this.config.audioRegions, this.config.sourceAudioFallbackPaths, this.config.sourceAudioFallbackStartDelayMsByPath, + this.config.sourceAudioTrackSettings, ), "audio processing", "audio", @@ -560,7 +575,9 @@ export class VideoExporter { speedRegions.length > 0 || audioRegions.length > 0 || sourceAudioFallbackPaths.length > 1 || - hasTimedSourceAudioFallback + hasTimedSourceAudioFallback || + hasNonDefaultSourceTrackSettings(this.config.sourceAudioTrackSettings) || + (this.config.clipRegions ?? []).some((clip) => Boolean(clip.muted)) ) { const sourceDurationMs = Math.max( 0, @@ -847,6 +864,8 @@ export class VideoExporter { this.config.audioRegions, this.config.sourceAudioFallbackPaths, this.config.sourceAudioFallbackStartDelayMsByPath, + this.config.sourceAudioTrackSettings, + this.config.clipRegions, ), "native edited audio rendering", "audio", @@ -943,6 +962,8 @@ export class VideoExporter { this.config.audioRegions, this.config.sourceAudioFallbackPaths, this.config.sourceAudioFallbackStartDelayMsByPath, + this.config.sourceAudioTrackSettings, + this.config.clipRegions, ), "ffmpeg edited audio rendering", "audio",