From 62f2a52860b52f26f3dcd7c22a39a3fe092e3b8d Mon Sep 17 00:00:00 2001 From: webadderall <131426131+webadderall@users.noreply.github.com> Date: Fri, 24 Apr 2026 12:04:21 +1000 Subject: [PATCH] Fix companion audio sync for trimmed and long exports --- electron/electron-env.d.ts | 10 +++- electron/ipc/recording/diagnostics.test.ts | 33 +++++++++++ electron/ipc/recording/diagnostics.ts | 57 +++++++++++++++++-- electron/ipc/register/recording.ts | 31 ++++++++-- electron/preload.ts | 8 ++- src/components/video-editor/VideoEditor.tsx | 33 ++++++++--- .../video-editor/timeline/TimelineEditor.tsx | 1 - src/hooks/useScreenRecorder.ts | 31 ++++++++-- src/lib/exporter/audioEncoder.test.ts | 52 +++++++++++++++++ src/lib/exporter/audioEncoder.ts | 19 ++++++- src/lib/exporter/modernVideoExporter.ts | 11 +++- src/lib/exporter/types.ts | 1 + src/lib/exporter/videoExporter.ts | 11 +++- src/lib/mediaTiming.test.ts | 10 +++- src/lib/mediaTiming.ts | 17 +++++- 15 files changed, 291 insertions(+), 34 deletions(-) diff --git a/electron/electron-env.d.ts b/electron/electron-env.d.ts index 5012dbe5..1f88763b 100644 --- a/electron/electron-env.d.ts +++ b/electron/electron-env.d.ts @@ -159,6 +159,9 @@ interface Window { storeMicrophoneSidecar: ( audioData: ArrayBuffer, videoPath: string, + options?: { + startDelayMs?: number; + }, ) => Promise<{ success: boolean; path?: string; error?: string }>; getRecordedVideoPath: () => Promise<{ success: boolean; path?: string; message?: string }>; listAssetDirectory: (relativeDir: string) => Promise<{ @@ -231,7 +234,12 @@ interface Window { }>; getVideoAudioFallbackPaths: ( videoPath: string, - ) => Promise<{ success: boolean; paths: string[]; error?: string }>; + ) => Promise<{ + success: boolean; + paths: string[]; + startDelayMsByPath?: Record; + error?: string; + }>; setRecordingState: (recording: boolean) => Promise; getCursorTelemetry: (videoPath?: string) => Promise<{ success: boolean; diff --git a/electron/ipc/recording/diagnostics.test.ts b/electron/ipc/recording/diagnostics.test.ts index b2a63085..9a775339 100644 --- a/electron/ipc/recording/diagnostics.test.ts +++ b/electron/ipc/recording/diagnostics.test.ts @@ -133,6 +133,39 @@ describe("getCompanionAudioFallbackPaths", () => { ]); }); + it("loads saved sidecar timing metadata alongside companion audio paths", async () => { + const videoPath = path.join(tempRoot, "recording.mp4"); + const micPath = path.join(tempRoot, "recording.mic.webm"); + + await Promise.all([ + fs.writeFile(videoPath, "video"), + fs.writeFile(micPath, "mic"), + fs.writeFile(`${micPath}.json`, JSON.stringify({ startDelayMs: 2750 })), + ]); + + execFileMock.mockImplementation( + ( + _file: string, + _args: string[], + _options: Record, + callback: ExecFileCallback, + ) => { + const error = new Error("ffmpeg probe failed") as Error & { stderr?: string }; + error.stderr = "Stream #0:0: Video: h264"; + callback(error, "", error.stderr); + }, + ); + + const { getCompanionAudioFallbackInfo } = await import("./diagnostics"); + + await expect(getCompanionAudioFallbackInfo(videoPath)).resolves.toEqual({ + paths: [micPath], + startDelayMsByPath: { + [micPath]: 2750, + }, + }); + }); + it("rejects tiny MP4 container-only outputs before they reach the editor", async () => { const videoPath = path.join(tempRoot, "recording-123.mp4"); await fs.writeFile(videoPath, Buffer.alloc(261)); diff --git a/electron/ipc/recording/diagnostics.ts b/electron/ipc/recording/diagnostics.ts index 1114414e..4588631e 100644 --- a/electron/ipc/recording/diagnostics.ts +++ b/electron/ipc/recording/diagnostics.ts @@ -9,6 +9,10 @@ import type { CompanionAudioCandidate, NativeCaptureDiagnostics } from "../types const execFileAsync = promisify(execFile); export const MIN_VALID_RECORDED_VIDEO_BYTES = 1024; +type CompanionAudioTimingMetadata = { + startDelayMs?: number; +}; + export function recordNativeCaptureDiagnostics( diagnostics: Omit, ) { @@ -99,6 +103,22 @@ export async function getUsableCompanionAudioCandidates( return candidates; } +async function readCompanionAudioTimingMetadata( + companionPath: string, +): Promise { + try { + const raw = await fs.readFile(`${companionPath}.json`, "utf8"); + const parsed = JSON.parse(raw) as CompanionAudioTimingMetadata | null; + if (!parsed || typeof parsed !== "object") { + return null; + } + + return parsed; + } catch { + return null; + } +} + export async function hasEmbeddedAudioStream(videoPath: string) { const ffmpegPath = getFfmpegBinaryPath(); let stderr = ""; @@ -118,11 +138,17 @@ export async function hasEmbeddedAudioStream(videoPath: string) { } export async function getCompanionAudioFallbackPaths(videoPath: string) { + const { paths } = await getCompanionAudioFallbackInfo(videoPath); + return paths; +} + +export async function getCompanionAudioFallbackInfo(videoPath: string) { const companionCandidates = await getUsableCompanionAudioCandidates(videoPath); if (companionCandidates.length === 0) { - return []; + return { paths: [], startDelayMsByPath: {} }; } + let paths: string[]; if (await hasEmbeddedAudioStream(videoPath)) { const microphoneCompanionPaths = Array.from( new Set( @@ -134,13 +160,36 @@ export async function getCompanionAudioFallbackPaths(videoPath: string) { ), ); if (microphoneCompanionPaths.length === 0) { - return []; + return { paths: [], startDelayMsByPath: {} }; } - return [videoPath, ...microphoneCompanionPaths]; + paths = [videoPath, ...microphoneCompanionPaths]; + } else { + paths = Array.from( + new Set(companionCandidates.flatMap((candidate) => candidate.usablePaths)), + ); } - return Array.from(new Set(companionCandidates.flatMap((candidate) => candidate.usablePaths))); + const metadataEntries = await Promise.all( + paths.map(async (audioPath) => { + const metadata = await readCompanionAudioTimingMetadata(audioPath); + const startDelayMs = metadata?.startDelayMs; + if (!Number.isFinite(startDelayMs) || (startDelayMs ?? 0) < 0) { + return null; + } + + return [audioPath, Math.round(startDelayMs ?? 0)] as const; + }), + ); + + return { + paths, + startDelayMsByPath: Object.fromEntries( + metadataEntries.filter( + (entry): entry is readonly [string, number] => entry !== null, + ), + ), + }; } export async function validateRecordedVideo(videoPath: string) { diff --git a/electron/ipc/register/recording.ts b/electron/ipc/register/recording.ts index 1f19cb51..9286efab 100644 --- a/electron/ipc/register/recording.ts +++ b/electron/ipc/register/recording.ts @@ -35,7 +35,7 @@ import { } from "../paths/binaries"; import { rememberApprovedLocalReadPath } from "../project/manager"; import { - getCompanionAudioFallbackPaths, + getCompanionAudioFallbackInfo, getFileSizeIfPresent, recordNativeCaptureDiagnostics, validateRecordedVideo, @@ -1013,19 +1013,19 @@ export function registerRecordingHandlers( ipcMain.handle("get-video-audio-fallback-paths", async (_event, videoPath: string) => { if (!videoPath) { - return { success: true, paths: [] }; + return { success: true, paths: [], startDelayMsByPath: {} }; } try { - const paths = await getCompanionAudioFallbackPaths(videoPath); + const { paths, startDelayMsByPath } = await getCompanionAudioFallbackInfo(videoPath); await Promise.all([ rememberApprovedLocalReadPath(videoPath), ...paths.map((fallbackPath) => rememberApprovedLocalReadPath(fallbackPath)), ]); - return { success: true, paths }; + return { success: true, paths, startDelayMsByPath }; } catch (error) { console.error("Failed to resolve companion audio fallback paths:", error); - return { success: false, paths: [], error: String(error) }; + return { success: false, paths: [], startDelayMsByPath: {}, error: String(error) }; } }); @@ -1170,11 +1170,30 @@ export function registerRecordingHandlers( ipcMain.handle( "store-microphone-sidecar", - async (_, audioData: ArrayBuffer, videoPath: string) => { + async ( + _, + audioData: ArrayBuffer, + videoPath: string, + options?: { startDelayMs?: number }, + ) => { try { const baseName = videoPath.replace(/\.[^.]+$/, ""); const sidecarPath = `${baseName}.mic.webm`; await fs.writeFile(sidecarPath, Buffer.from(audioData)); + const startDelayMs = options?.startDelayMs; + if (Number.isFinite(startDelayMs) && (startDelayMs ?? 0) >= 0) { + try { + await fs.writeFile( + `${sidecarPath}.json`, + JSON.stringify({ startDelayMs: Math.round(startDelayMs ?? 0) }), + ); + } catch (metadataError) { + console.warn( + "Failed to store microphone sidecar timing metadata:", + metadataError, + ); + } + } return { success: true, path: sidecarPath }; } catch (error) { console.error("Failed to store microphone sidecar:", error); diff --git a/electron/preload.ts b/electron/preload.ts index a8b0e67c..a47c13e5 100644 --- a/electron/preload.ts +++ b/electron/preload.ts @@ -268,8 +268,12 @@ contextBridge.exposeInMainWorld("electronAPI", { storeRecordedVideo: (videoData: ArrayBuffer, fileName: string) => { return ipcRenderer.invoke("store-recorded-video", videoData, fileName); }, - storeMicrophoneSidecar: (audioData: ArrayBuffer, videoPath: string) => { - return ipcRenderer.invoke("store-microphone-sidecar", audioData, videoPath); + storeMicrophoneSidecar: ( + audioData: ArrayBuffer, + videoPath: string, + options?: { startDelayMs?: number }, + ) => { + return ipcRenderer.invoke("store-microphone-sidecar", audioData, videoPath, options); }, getRecordedVideoPath: () => { return ipcRenderer.invoke("get-recorded-video-path"); diff --git a/src/components/video-editor/VideoEditor.tsx b/src/components/video-editor/VideoEditor.tsx index a436aa25..6e4e7b98 100644 --- a/src/components/video-editor/VideoEditor.tsx +++ b/src/components/video-editor/VideoEditor.tsx @@ -596,6 +596,8 @@ export default function VideoEditor() { const [showExportDropdown, setShowExportDropdown] = useState(false); const [previewVolume, setPreviewVolume] = useState(1); const [sourceAudioFallbackPaths, setSourceAudioFallbackPaths] = useState([]); + const [sourceAudioFallbackStartDelayMsByPath, setSourceAudioFallbackStartDelayMsByPath] = + useState>({}); const [aspectRatio, setAspectRatio] = useState( initialEditorPreferences.aspectRatio, ); @@ -1219,6 +1221,7 @@ export default function VideoEditor() { useEffect(() => { let cancelled = false; setSourceAudioFallbackPaths([]); + setSourceAudioFallbackStartDelayMsByPath({}); if (!currentSourcePath) { return () => { @@ -1235,6 +1238,7 @@ export default function VideoEditor() { } if (!result.success) { setSourceAudioFallbackPaths([]); + setSourceAudioFallbackStartDelayMsByPath({}); toast.warning( result.error ? `Could not load companion audio sources: ${summarizeErrorMessage(result.error)}` @@ -1246,9 +1250,11 @@ export default function VideoEditor() { toast.dismiss(SOURCE_AUDIO_FALLBACK_TOAST_ID); setSourceAudioFallbackPaths(result.paths ?? []); + setSourceAudioFallbackStartDelayMsByPath(result.startDelayMsByPath ?? {}); } catch (error) { if (!cancelled) { setSourceAudioFallbackPaths([]); + setSourceAudioFallbackStartDelayMsByPath({}); toast.warning( `Could not load companion audio sources: ${summarizeErrorMessage(String(error))}`, { id: SOURCE_AUDIO_FALLBACK_TOAST_ID, duration: 10000 }, @@ -3516,13 +3522,14 @@ export default function VideoEditor() { } } - for (const audioPath of previewSourceAudioFallbackPaths) { - let audio = existing.get(audioPath); - if (!audio) { - audio = new Audio(); - audio.preload = "auto"; - existing.set(audioPath, audio); - } + for (const audioPath of previewSourceAudioFallbackPaths) { + let audio = existing.get(audioPath); + if (!audio) { + audio = new Audio(); + audio.preload = "auto"; + existing.set(audioPath, audio); + } + audio.dataset.sourceAudioPath = audioPath; if (sourceAudioElementResourcesRef.current.get(audioPath) !== audioPath) { audio.pause(); @@ -3665,6 +3672,7 @@ export default function VideoEditor() { const startDelaySeconds = estimateCompanionAudioStartDelaySeconds( duration, audioDuration, + sourceAudioFallbackStartDelayMsByPath[audio.dataset.sourceAudioPath ?? ""], ); const beforeAudioStart = currentTime + 0.001 < startDelaySeconds; const targetTime = clampMediaTimeToDuration( @@ -3698,7 +3706,14 @@ export default function VideoEditor() { } lastSourceAudioSyncTimeRef.current = currentTime; - }, [currentTime, duration, isPlaying, previewSourceAudioFallbackPaths, speedRegions]); + }, [ + currentTime, + duration, + isPlaying, + previewSourceAudioFallbackPaths, + sourceAudioFallbackStartDelayMsByPath, + speedRegions, + ]); const showExportSuccessToast = useCallback((filePath: string) => { toast.success(`Exported successfully to ${filePath}`, { @@ -4018,6 +4033,7 @@ export default function VideoEditor() { frame, audioRegions, sourceAudioFallbackPaths, + sourceAudioFallbackStartDelayMsByPath, previewWidth, previewHeight, onProgress: (progress: ExportProgress) => { @@ -4216,6 +4232,7 @@ export default function VideoEditor() { cursorSway, audioRegions, sourceAudioFallbackPaths, + sourceAudioFallbackStartDelayMsByPath, exportEncodingMode, exportBackendPreference, exportPipelineModel, diff --git a/src/components/video-editor/timeline/TimelineEditor.tsx b/src/components/video-editor/timeline/TimelineEditor.tsx index 159e98fe..b4d3d351 100644 --- a/src/components/video-editor/timeline/TimelineEditor.tsx +++ b/src/components/video-editor/timeline/TimelineEditor.tsx @@ -44,7 +44,6 @@ import { } from "@/utils/aspectRatioUtils"; import { formatShortcut } from "@/utils/platformUtils"; import { loadEditorPreferences, saveEditorPreferences } from "../editorPreferences"; -import { toFileUrl } from "../projectPersistence"; import type { AnnotationRegion, AudioRegion, diff --git a/src/hooks/useScreenRecorder.ts b/src/hooks/useScreenRecorder.ts index 1b56dad9..cb41930e 100644 --- a/src/hooks/useScreenRecorder.ts +++ b/src/hooks/useScreenRecorder.ts @@ -152,6 +152,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { const recordingFinalizationToastId = useRef(null); const micFallbackRecorder = useRef(null); const micFallbackChunks = useRef([]); + const micFallbackStartDelayMs = useRef(null); const showRecordingFinalizationToast = useCallback((message = "Preparing recording...") => { recordingFinalizationToastId.current = toast.loading(message, { @@ -472,6 +473,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { async ( micFallbackBlobPromise: Promise | null | undefined, finalPath: string, + startDelayMs?: number | null, ) => { const micFallbackBlob = await micFallbackBlobPromise; if (!micFallbackBlob) { @@ -483,6 +485,9 @@ export function useScreenRecorder(): UseScreenRecorderReturn { const result = await window.electronAPI.storeMicrophoneSidecar( arrayBuffer, finalPath, + Number.isFinite(startDelayMs) && (startDelayMs ?? 0) >= 0 + ? { startDelayMs: startDelayMs ?? 0 } + : undefined, ); if (!result.success) { const errorMessage = @@ -531,7 +536,10 @@ export function useScreenRecorder(): UseScreenRecorderReturn { }, []); const recoverNativeRecordingSession = useCallback( - async (micFallbackBlobPromise?: Promise | null) => { + async ( + micFallbackBlobPromise?: Promise | null, + startDelayMs?: number | null, + ) => { if (typeof window.electronAPI?.recoverNativeScreenRecording !== "function") { return null; } @@ -544,7 +552,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { const resolvedMicFallbackBlobPromise = micFallbackBlobPromise ?? stopMicFallbackRecorder(); const webcamPath = await stopWebcamRecorder(); - await storeMicrophoneSidecar(resolvedMicFallbackBlobPromise, result.path); + await storeMicrophoneSidecar(resolvedMicFallbackBlobPromise, result.path, startDelayMs); await finalizeRecordingSession(result.path, webcamPath); return result.path; }, @@ -685,6 +693,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { void (async () => { showRecordingFinalizationToast(); + const fallbackStartDelayMs = micFallbackStartDelayMs.current; const micFallbackBlobPromise = stopMicFallbackRecorder(); const webcamPath = await stopWebcamRecorder(); const isNativeWindows = nativeWindowsRecording.current; @@ -702,8 +711,10 @@ export function useScreenRecorder(): UseScreenRecorderReturn { ); void logNativeCaptureDiagnostics("stop-native-screen-recording"); try { - const recoveredPath = - await recoverNativeRecordingSession(micFallbackBlobPromise); + const recoveredPath = await recoverNativeRecordingSession( + micFallbackBlobPromise, + fallbackStartDelayMs, + ); if (recoveredPath) { return; } @@ -750,7 +761,11 @@ export function useScreenRecorder(): UseScreenRecorderReturn { finalPath = muxResult.path; } - await storeMicrophoneSidecar(micFallbackBlobPromise, finalPath); + await storeMicrophoneSidecar( + micFallbackBlobPromise, + finalPath, + fallbackStartDelayMs, + ); await finalizeRecordingSession(finalPath, webcamPath); })(); @@ -1025,6 +1040,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { if (nativeResult.success) { const mainStartedAt = Date.now(); + micFallbackStartDelayMs.current = null; beginWebcamCapture(); nativeScreenRecording.current = true; nativeWindowsRecording.current = useNativeWindowsCapture; @@ -1067,9 +1083,14 @@ export function useScreenRecorder(): UseScreenRecorderReturn { micFallbackChunks.current.push(event.data); } }; + micFallbackStartDelayMs.current = Math.max( + 0, + Date.now() - mainStartedAt, + ); recorder.start(RECORDER_TIMESLICE_MS); micFallbackRecorder.current = recorder; } catch (micError) { + micFallbackStartDelayMs.current = null; console.warn("Browser microphone fallback failed:", micError); const permissionDenied = micError instanceof DOMException && diff --git a/src/lib/exporter/audioEncoder.test.ts b/src/lib/exporter/audioEncoder.test.ts index a39e9358..9e9deeec 100644 --- a/src/lib/exporter/audioEncoder.test.ts +++ b/src/lib/exporter/audioEncoder.test.ts @@ -12,6 +12,7 @@ type OfflineRenderTestHarness = AudioProcessor & { speedRegions: never[], audioRegions: never[], sourceAudioFallbackPaths: string[], + sourceAudioFallbackStartDelayMsByPath?: Record, ): Promise<{ mainBuffer: AudioBuffer | null; companionEntries: Array<{ buffer: AudioBuffer; startDelaySec: number }>; @@ -22,6 +23,7 @@ type OfflineRenderTestHarness = AudioProcessor & { speedRegions: never[], audioRegions: never[], sourceAudioFallbackPaths: string[], + sourceAudioFallbackStartDelayMsByPath: Record | undefined, muxer: unknown, ): Promise; }; @@ -82,4 +84,54 @@ describe("AudioProcessor offline render preparation", () => { expect(loadAudioFileDemuxer).not.toHaveBeenCalled(); expect(renderAndMuxOfflineAudio).not.toHaveBeenCalled(); }); + + it("uses recorded companion start-delay metadata instead of inferring from duration gap", async () => { + const processor = new AudioProcessor() as unknown as OfflineRenderTestHarness; + const mainBuffer = { duration: 600, numberOfChannels: 2 } as AudioBuffer; + const micBuffer = { duration: 565, numberOfChannels: 1 } as AudioBuffer; + + vi.spyOn(processor, "decodeAudioFromUrl").mockImplementation(async (url: string) => { + if (url === "file:///tmp/recording.mp4") { + return mainBuffer; + } + if (url === "/tmp/recording.mic.webm") { + return micBuffer; + } + return null; + }); + + const prepared = await processor.prepareOfflineRender( + "file:///tmp/recording.mp4", + [], + [], + [], + ["/tmp/recording.mic.webm"], + { "/tmp/recording.mic.webm": 3_500 }, + ); + + expect(prepared.companionEntries[0]?.startDelaySec).toBeCloseTo(3.5); + }); + + it("avoids the single-sidecar fast path when companion timing metadata is present", async () => { + const processor = new AudioProcessor() as unknown as OfflineRenderTestHarness; + const loadAudioFileDemuxer = vi.spyOn(processor, "loadAudioFileDemuxer"); + const renderAndMuxOfflineAudio = vi + .spyOn(processor, "renderAndMuxOfflineAudio") + .mockResolvedValue(); + + await processor.process( + null, + {} as never, + "file:///tmp/recording.mp4", + [], + [], + undefined, + [], + ["/tmp/recording.mic.webm"], + { "/tmp/recording.mic.webm": 2_000 }, + ); + + expect(loadAudioFileDemuxer).not.toHaveBeenCalled(); + expect(renderAndMuxOfflineAudio).toHaveBeenCalled(); + }); }); diff --git a/src/lib/exporter/audioEncoder.ts b/src/lib/exporter/audioEncoder.ts index 150fbfea..d78d2cbc 100644 --- a/src/lib/exporter/audioEncoder.ts +++ b/src/lib/exporter/audioEncoder.ts @@ -143,6 +143,7 @@ export class AudioProcessor { readEndSec?: number, audioRegions?: AudioRegion[], sourceAudioFallbackPaths?: string[], + sourceAudioFallbackStartDelayMsByPath?: Record, ): Promise { const sortedTrims = trimRegions ? [...trimRegions].sort((a, b) => a.startMs - b.startMs) @@ -164,9 +165,13 @@ export class AudioProcessor { videoUrl, sortedSourceAudioFallbackPaths, ); + const hasTimedCompanionAudio = externalAudioPaths.some( + (audioPath) => (sourceAudioFallbackStartDelayMsByPath?.[audioPath] ?? 0) > 0, + ); const needsSourceAudioMixing = externalAudioPaths.length > 1 || - (hasEmbeddedSourceAudio && externalAudioPaths.length > 0); + (hasEmbeddedSourceAudio && externalAudioPaths.length > 0) || + hasTimedCompanionAudio; // When speed edits, audio regions, or multiple audio sources need mixing, use offline AudioContext pipeline. if ( @@ -180,6 +185,7 @@ export class AudioProcessor { sortedSpeedRegions, sortedAudioRegions, sortedSourceAudioFallbackPaths, + sourceAudioFallbackStartDelayMsByPath, muxer, ); return; @@ -210,6 +216,7 @@ export class AudioProcessor { [], [], externalAudioPaths, + sourceAudioFallbackStartDelayMsByPath, muxer, ); return; @@ -255,6 +262,7 @@ export class AudioProcessor { speedRegions?: SpeedRegion[], audioRegions?: AudioRegion[], sourceAudioFallbackPaths?: string[], + sourceAudioFallbackStartDelayMsByPath?: Record, ): Promise { const sortedTrims = trimRegions ? [...trimRegions].sort((a, b) => a.startMs - b.startMs) @@ -279,6 +287,7 @@ export class AudioProcessor { sortedSpeedRegions, sortedAudioRegions, sortedSourceAudioFallbackPaths, + sourceAudioFallbackStartDelayMsByPath, ); return this.renderToWavBlobChunked(prepared); } @@ -528,6 +537,7 @@ export class AudioProcessor { speedRegions: SpeedRegion[], audioRegions: AudioRegion[], sourceAudioFallbackPaths: string[], + sourceAudioFallbackStartDelayMsByPath: Record | undefined, muxer: VideoMuxer, ): Promise { const prepared = await this.prepareOfflineRender( @@ -536,6 +546,7 @@ export class AudioProcessor { speedRegions, audioRegions, sourceAudioFallbackPaths, + sourceAudioFallbackStartDelayMsByPath, ); if (this.cancelled) return; await this.renderAndEncodeChunked(prepared, muxer); @@ -547,6 +558,7 @@ export class AudioProcessor { speedRegions: SpeedRegion[], audioRegions: AudioRegion[], sourceAudioFallbackPaths: string[], + sourceAudioFallbackStartDelayMsByPath?: Record, ): Promise { if (this.cancelled) throw new Error("Export cancelled"); this.onProgress?.(0); @@ -562,17 +574,20 @@ export class AudioProcessor { // Decode companion / sidecar audio files const companionEntries: Array<{ buffer: AudioBuffer; startDelaySec: number }> = []; + const refDuration = + mainBuffer?.duration ?? + (externalAudioPaths.length > 0 ? await this.getMediaDurationSec(videoUrl) : 0); for (const audioPath of externalAudioPaths) { if (this.cancelled) throw new Error("Export cancelled"); const buffer = await this.decodeAudioFromUrl(audioPath); if (!buffer) continue; - const refDuration = mainBuffer?.duration ?? (await this.getMediaDurationSec(videoUrl)); companionEntries.push({ buffer, startDelaySec: estimateCompanionAudioStartDelaySeconds( refDuration, buffer.duration, + sourceAudioFallbackStartDelayMsByPath?.[audioPath], ), }); } diff --git a/src/lib/exporter/modernVideoExporter.ts b/src/lib/exporter/modernVideoExporter.ts index 7b798d74..52b38441 100644 --- a/src/lib/exporter/modernVideoExporter.ts +++ b/src/lib/exporter/modernVideoExporter.ts @@ -91,6 +91,7 @@ interface VideoExporterConfig extends ExportConfig { frame?: string | null; audioRegions?: AudioRegion[]; sourceAudioFallbackPaths?: string[]; + sourceAudioFallbackStartDelayMsByPath?: Record; previewWidth?: number; previewHeight?: number; onProgress?: (progress: ExportProgress) => void; @@ -497,6 +498,7 @@ export class ModernVideoExporter { undefined, this.config.audioRegions, this.config.sourceAudioFallbackPaths, + this.config.sourceAudioFallbackStartDelayMsByPath, ), "audio processing", "audio", @@ -728,6 +730,10 @@ export class ModernVideoExporter { const sourceAudioFallbackPaths = (this.config.sourceAudioFallbackPaths ?? []).filter( (audioPath) => typeof audioPath === "string" && audioPath.trim().length > 0, ); + const hasTimedSourceAudioFallback = sourceAudioFallbackPaths.some( + (audioPath) => + (this.config.sourceAudioFallbackStartDelayMsByPath?.[audioPath] ?? 0) > 0, + ); const localVideoSourcePath = this.getNativeVideoSourcePath(); const primaryAudioSourcePath = (videoInfo.hasAudio ? localVideoSourcePath : null) ?? @@ -745,7 +751,8 @@ export class ModernVideoExporter { if ( speedRegions.length > 0 || audioRegions.length > 0 || - sourceAudioFallbackPaths.length > 1 + sourceAudioFallbackPaths.length > 1 || + hasTimedSourceAudioFallback ) { const sourceDurationMs = Math.max( 0, @@ -1026,6 +1033,7 @@ export class ModernVideoExporter { this.config.speedRegions, this.config.audioRegions, this.config.sourceAudioFallbackPaths, + this.config.sourceAudioFallbackStartDelayMsByPath, ), `${NATIVE_EXPORT_ENGINE_NAME} edited audio rendering`, "audio", @@ -1137,6 +1145,7 @@ export class ModernVideoExporter { this.config.speedRegions, this.config.audioRegions, this.config.sourceAudioFallbackPaths, + this.config.sourceAudioFallbackStartDelayMsByPath, ), "FFmpeg edited audio rendering", "audio", diff --git a/src/lib/exporter/types.ts b/src/lib/exporter/types.ts index 85df6690..937f34ec 100644 --- a/src/lib/exporter/types.ts +++ b/src/lib/exporter/types.ts @@ -11,6 +11,7 @@ export interface ExportConfig { maxDecodeQueue?: number; maxPendingFrames?: number; maxInFlightNativeWrites?: number; + sourceAudioFallbackStartDelayMsByPath?: Record; } export type ExportRenderBackend = "webgpu" | "webgl"; diff --git a/src/lib/exporter/videoExporter.ts b/src/lib/exporter/videoExporter.ts index 7a6c05f5..452d64ab 100644 --- a/src/lib/exporter/videoExporter.ts +++ b/src/lib/exporter/videoExporter.ts @@ -79,6 +79,7 @@ interface VideoExporterConfig extends ExportConfig { frame?: string | null; audioRegions?: AudioRegion[]; sourceAudioFallbackPaths?: string[]; + sourceAudioFallbackStartDelayMsByPath?: Record; previewWidth?: number; previewHeight?: number; onProgress?: (progress: ExportProgress) => void; @@ -369,6 +370,7 @@ export class VideoExporter { undefined, this.config.audioRegions, this.config.sourceAudioFallbackPaths, + this.config.sourceAudioFallbackStartDelayMsByPath, ), "audio processing", "audio", @@ -495,6 +497,10 @@ export class VideoExporter { const sourceAudioFallbackPaths = (this.config.sourceAudioFallbackPaths ?? []).filter( (audioPath) => typeof audioPath === "string" && audioPath.trim().length > 0, ); + const hasTimedSourceAudioFallback = sourceAudioFallbackPaths.some( + (audioPath) => + (this.config.sourceAudioFallbackStartDelayMsByPath?.[audioPath] ?? 0) > 0, + ); const localVideoSourcePath = this.getNativeVideoSourcePath(); const primaryAudioSourcePath = (videoInfo.hasAudio ? localVideoSourcePath : null) ?? @@ -512,7 +518,8 @@ export class VideoExporter { if ( speedRegions.length > 0 || audioRegions.length > 0 || - sourceAudioFallbackPaths.length > 1 + sourceAudioFallbackPaths.length > 1 || + hasTimedSourceAudioFallback ) { const sourceDurationMs = Math.max( 0, @@ -786,6 +793,7 @@ export class VideoExporter { this.config.speedRegions, this.config.audioRegions, this.config.sourceAudioFallbackPaths, + this.config.sourceAudioFallbackStartDelayMsByPath, ), "native edited audio rendering", "audio", @@ -884,6 +892,7 @@ export class VideoExporter { this.config.speedRegions, this.config.audioRegions, this.config.sourceAudioFallbackPaths, + this.config.sourceAudioFallbackStartDelayMsByPath, ), "ffmpeg edited audio rendering", "audio", diff --git a/src/lib/mediaTiming.test.ts b/src/lib/mediaTiming.test.ts index 62885863..a6f20479 100644 --- a/src/lib/mediaTiming.test.ts +++ b/src/lib/mediaTiming.test.ts @@ -21,14 +21,20 @@ describe("clampMediaTimeToDuration", () => { }); describe("estimateCompanionAudioStartDelaySeconds", () => { - it("returns the positive tail gap when companion audio is shorter", () => { + it("keeps small inferred offsets when the companion audio is only slightly shorter", () => { expect(estimateCompanionAudioStartDelaySeconds(10, 9.6)).toBeCloseTo(0.4); expect(estimateCompanionAudioStartDelaySeconds(10, 9.97)).toBeCloseTo(0.03); }); - it("ignores tiny or negative differences", () => { + it("prefers an explicitly recorded start delay", () => { + expect(estimateCompanionAudioStartDelaySeconds(10, 2, 3_500)).toBeCloseTo(3.5); + expect(estimateCompanionAudioStartDelaySeconds(10, 2, 0)).toBe(0); + }); + + it("ignores tiny, negative, or suspiciously large inferred differences", () => { expect(estimateCompanionAudioStartDelaySeconds(10, 9.99)).toBe(0); expect(estimateCompanionAudioStartDelaySeconds(10, 10.5)).toBe(0); + expect(estimateCompanionAudioStartDelaySeconds(600, 565)).toBe(0); }); }); diff --git a/src/lib/mediaTiming.ts b/src/lib/mediaTiming.ts index ed3c0ff4..d9dcf673 100644 --- a/src/lib/mediaTiming.ts +++ b/src/lib/mediaTiming.ts @@ -7,10 +7,18 @@ export function clampMediaTimeToDuration(targetTime: number, duration?: number | return Math.max(0, Math.min(safeTargetTime, Math.max(0, duration))); } +const MIN_COMPANION_AUDIO_DELAY_SECONDS = 0.025; +const MAX_INFERRED_COMPANION_AUDIO_DELAY_SECONDS = 0.5; + export function estimateCompanionAudioStartDelaySeconds( timelineDuration?: number | null, audioDuration?: number | null, + recordedStartDelayMs?: number | null, ): number { + if (Number.isFinite(recordedStartDelayMs) && (recordedStartDelayMs ?? 0) >= 0) { + return Math.max(0, recordedStartDelayMs ?? 0) / 1000; + } + if (!Number.isFinite(timelineDuration) || !Number.isFinite(audioDuration)) { return 0; } @@ -19,7 +27,14 @@ export function estimateCompanionAudioStartDelaySeconds( const safeAudioDuration = Math.max(0, audioDuration ?? 0); const estimatedDelaySeconds = safeTimelineDuration - safeAudioDuration; - return estimatedDelaySeconds > 0.025 ? estimatedDelaySeconds : 0; + if ( + estimatedDelaySeconds <= MIN_COMPANION_AUDIO_DELAY_SECONDS || + estimatedDelaySeconds > MAX_INFERRED_COMPANION_AUDIO_DELAY_SECONDS + ) { + return 0; + } + + return estimatedDelaySeconds; } export function getMediaSyncPlaybackRate({