Add sidecar audio fallback for silent recordings

src/components/video-editor/VideoEditor.tsx

@@ -43,6 +43,7 @@ import {
 	VideoExporter,
 } from "@/lib/exporter";
 import { resolveMediaElementSource } from "@/lib/exporter/localMediaSource";
+import { clampMediaTimeToDuration } from "@/lib/mediaTiming";
 import { matchesShortcut } from "@/lib/shortcuts";
 import { type AspectRatio, getAspectRatioValue } from "@/utils/aspectRatioUtils";
 import { resolveAutoCaptionSourcePath } from "./autoCaptionSource";
@@ -398,6 +399,7 @@ export default function VideoEditor() {
 	const [exportError, setExportError] = useState<string | null>(null);
 	const [showExportDropdown, setShowExportDropdown] = useState(false);
 	const [previewVolume, setPreviewVolume] = useState(1);
+	const [sourceAudioFallbackPaths, setSourceAudioFallbackPaths] = useState<string[]>([]);
 	const [aspectRatio, setAspectRatio] = useState<AspectRatio>(initialEditorPreferences.aspectRatio);
 	const [activeEffectSection, setActiveEffectSection] = useState<EditorEffectSection>("scene");
 	const [exportQuality, setExportQuality] = useState<ExportQuality>(
@@ -881,6 +883,36 @@ export default function VideoEditor() {
 		() => videoSourcePath ?? (videoPath ? fromFileUrl(videoPath) : null),
 		[videoPath, videoSourcePath],
 	);
+	const hasSourceAudioFallback = sourceAudioFallbackPaths.length > 0;
+
+	useEffect(() => {
+		let cancelled = false;
+		setSourceAudioFallbackPaths([]);
+
+		if (!currentSourcePath) {
+			return () => {
+				cancelled = true;
+			};
+		}
+
+		void (async () => {
+			try {
+				const result = await window.electronAPI.getVideoAudioFallbackPaths(currentSourcePath);
+				if (cancelled) {
+					return;
+				}
+				setSourceAudioFallbackPaths(result.success ? (result.paths ?? []) : []);
+			} catch {
+				if (!cancelled) {
+					setSourceAudioFallbackPaths([]);
+				}
+			}
+		})();
+
+		return () => {
+			cancelled = true;
+		};
+	}, [currentSourcePath]);
 
 	const projectDisplayName = useMemo(() => {
 		const fileName =
@@ -2374,6 +2406,10 @@ export default function VideoEditor() {
 	const audioElementsRef = useRef<Map<string, HTMLAudioElement>>(new Map());
 	const audioElementRevokersRef = useRef<Map<string, () => void>>(new Map());
 	const audioElementResourcesRef = useRef<Map<string, string>>(new Map());
+	const sourceAudioElementsRef = useRef<Map<string, HTMLAudioElement>>(new Map());
+	const sourceAudioElementRevokersRef = useRef<Map<string, () => void>>(new Map());
+	const sourceAudioElementResourcesRef = useRef<Map<string, string>>(new Map());
+	const lastSourceAudioSyncTimeRef = useRef<number | null>(null);
 
 	useEffect(() => {
 		let cancelled = false;
@@ -2434,6 +2470,67 @@ export default function VideoEditor() {
 		};
 	}, [audioRegions, previewVolume]);
 
+	useEffect(() => {
+		let cancelled = false;
+		const existing = sourceAudioElementsRef.current;
+		const currentIds = new Set(sourceAudioFallbackPaths);
+
+		for (const [id, audio] of existing) {
+			if (!currentIds.has(id)) {
+				audio.pause();
+				audio.src = "";
+				sourceAudioElementRevokersRef.current.get(id)?.();
+				sourceAudioElementRevokersRef.current.delete(id);
+				sourceAudioElementResourcesRef.current.delete(id);
+				existing.delete(id);
+			}
+		}
+
+		for (const audioPath of sourceAudioFallbackPaths) {
+			let audio = existing.get(audioPath);
+			if (!audio) {
+				audio = new Audio();
+				audio.preload = "auto";
+				existing.set(audioPath, audio);
+			}
+
+			if (sourceAudioElementResourcesRef.current.get(audioPath) !== audioPath) {
+				audio.pause();
+				audio.src = "";
+				sourceAudioElementRevokersRef.current.get(audioPath)?.();
+				sourceAudioElementRevokersRef.current.delete(audioPath);
+				sourceAudioElementResourcesRef.current.set(audioPath, audioPath);
+
+				void (async () => {
+					const resolved = await resolveMediaElementSource(audioPath);
+					const latestAudio = existing.get(audioPath);
+
+					if (
+						cancelled ||
+						latestAudio !== audio ||
+						sourceAudioElementResourcesRef.current.get(audioPath) !== audioPath
+					) {
+						resolved.revoke();
+						return;
+					}
+
+					sourceAudioElementRevokersRef.current.set(audioPath, resolved.revoke);
+					latestAudio.src = resolved.src;
+				})();
+			}
+
+			audio.volume = Math.max(0, Math.min(1, previewVolume));
+		}
+
+		if (sourceAudioFallbackPaths.length === 0) {
+			lastSourceAudioSyncTimeRef.current = null;
+		}
+
+		return () => {
+			cancelled = true;
+		};
+	}, [previewVolume, sourceAudioFallbackPaths]);
+
 	useEffect(() => {
 		return () => {
 			for (const audio of audioElementsRef.current.values()) {
@@ -2446,6 +2543,17 @@ export default function VideoEditor() {
 			audioElementsRef.current.clear();
 			audioElementRevokersRef.current.clear();
 			audioElementResourcesRef.current.clear();
+			for (const audio of sourceAudioElementsRef.current.values()) {
+				audio.pause();
+				audio.src = "";
+			}
+			for (const revoke of sourceAudioElementRevokersRef.current.values()) {
+				revoke();
+			}
+			sourceAudioElementsRef.current.clear();
+			sourceAudioElementRevokersRef.current.clear();
+			sourceAudioElementResourcesRef.current.clear();
+			lastSourceAudioSyncTimeRef.current = null;
 		};
 	}, []);
 
@@ -2475,6 +2583,50 @@ export default function VideoEditor() {
 		}
 	}, [isPlaying, currentTime, audioRegions]);
 
+	useEffect(() => {
+		if (sourceAudioFallbackPaths.length === 0) {
+			lastSourceAudioSyncTimeRef.current = null;
+			return;
+		}
+
+		const activeSpeedRegion = speedRegions.find(
+			(region) => currentTime * 1000 >= region.startMs && currentTime * 1000 < region.endMs,
+		);
+		const targetPlaybackRate = activeSpeedRegion ? activeSpeedRegion.speed : 1;
+		const previousTimelineTime = lastSourceAudioSyncTimeRef.current;
+		const timelineJumped =
+			previousTimelineTime === null || Math.abs(currentTime - previousTimelineTime) > 0.25;
+		const driftThreshold = isPlaying ? 0.35 : 0.01;
+
+		for (const audio of sourceAudioElementsRef.current.values()) {
+			const targetTime = clampMediaTimeToDuration(
+				currentTime,
+				Number.isFinite(audio.duration) ? audio.duration : null,
+			);
+
+			if (Math.abs(audio.playbackRate - targetPlaybackRate) > 0.001) {
+				audio.playbackRate = targetPlaybackRate;
+			}
+
+			if (timelineJumped || Math.abs(audio.currentTime - targetTime) > driftThreshold) {
+				try {
+					audio.currentTime = targetTime;
+				} catch {
+					// no-op
+				}
+			}
+
+			const atEnd = Number.isFinite(audio.duration) && targetTime >= audio.duration;
+			if (isPlaying && !atEnd) {
+				audio.play().catch(() => undefined);
+			} else if (!audio.paused) {
+				audio.pause();
+			}
+		}
+
+		lastSourceAudioSyncTimeRef.current = currentTime;
+	}, [currentTime, isPlaying, sourceAudioFallbackPaths, speedRegions]);
+
 	const showExportSuccessToast = useCallback((filePath: string) => {
 		toast.success(`Exported successfully to ${filePath}`, {
 			action: {
@@ -2683,6 +2835,7 @@ export default function VideoEditor() {
 						cursorClickBounceDuration,
 						cursorSway,
 						audioRegions,
+						sourceAudioFallbackPaths,
 						previewWidth,
 						previewHeight,
 						onProgress: (progress: ExportProgress) => {
@@ -3346,7 +3499,7 @@ export default function VideoEditor() {
 												cursorClickBounce={cursorClickBounce}
 												cursorClickBounceDuration={cursorClickBounceDuration}
 												cursorSway={cursorSway}
-												volume={previewVolume}
+												volume={hasSourceAudioFallback ? 0 : previewVolume}
 											/>
 										</div>
 									</div>

src/lib/exporter/audioEncoder.ts

@@ -18,13 +18,14 @@ export class AudioProcessor {
    * 2) speed regions present -> pitch-preserving rendered timeline pipeline
    */
   async process(
-    demuxer: WebDemuxer,
+    demuxer: WebDemuxer | null,
     muxer: VideoMuxer,
     videoUrl: string,
     trimRegions?: TrimRegion[],
     speedRegions?: SpeedRegion[],
     readEndSec?: number,
     audioRegions?: AudioRegion[],
+    sourceAudioFallbackPaths?: string[],
   ): Promise<void> {
     const sortedTrims = trimRegions ? [...trimRegions].sort((a, b) => a.startMs - b.startMs) : []
     const sortedSpeedRegions = speedRegions
@@ -35,14 +36,22 @@ export class AudioProcessor {
     const sortedAudioRegions = audioRegions
       ? [...audioRegions].sort((a, b) => a.startMs - b.startMs)
       : []
+    const sortedSourceAudioFallbackPaths = sourceAudioFallbackPaths
+      ? sourceAudioFallbackPaths.filter((audioPath) => typeof audioPath === 'string' && audioPath.trim().length > 0)
+      : []
 
     // When audio regions or speed edits are present, use AudioContext mixing path.
-    if (sortedSpeedRegions.length > 0 || sortedAudioRegions.length > 0) {
+    if (
+      sortedSpeedRegions.length > 0
+      || sortedAudioRegions.length > 0
+      || sortedSourceAudioFallbackPaths.length > 0
+    ) {
       const renderedAudioBlob = await this.renderMixedTimelineAudio(
         videoUrl,
         sortedTrims,
         sortedSpeedRegions,
         sortedAudioRegions,
+        sortedSourceAudioFallbackPaths,
       )
       if (!this.cancelled) {
         await this.muxRenderedAudioBlob(renderedAudioBlob, muxer)
@@ -51,6 +60,11 @@ export class AudioProcessor {
     }
 
     // No speed edits or audio regions: keep the original demux/decode/encode path with trim timestamp remap.
+    if (!demuxer) {
+      console.warn('[AudioProcessor] No demuxer available, skipping audio')
+      return
+    }
+
     await this.processTrimOnlyAudio(demuxer, muxer, sortedTrims, readEndSec)
   }
 
@@ -279,13 +293,15 @@ export class AudioProcessor {
     trimRegions: TrimRegion[],
     speedRegions: SpeedRegion[],
     audioRegions: AudioRegion[],
+    sourceAudioFallbackPaths: string[] = [],
   ): Promise<Blob> {
-    const mediaSource = await resolveMediaElementSource(videoUrl)
-    const media = document.createElement('audio')
-    media.src = mediaSource.src
-    media.preload = 'auto'
+    const timelineMediaSource = await resolveMediaElementSource(videoUrl)
+    const timelineMedia = document.createElement('video')
+    timelineMedia.src = timelineMediaSource.src
+    timelineMedia.preload = 'auto'
+    timelineMedia.playsInline = true
 
-    const pitchMedia = media as HTMLMediaElement & {
+    const pitchMedia = timelineMedia as HTMLMediaElement & {
       preservesPitch?: boolean
       mozPreservesPitch?: boolean
       webkitPreservesPitch?: boolean
@@ -294,7 +310,7 @@ export class AudioProcessor {
     pitchMedia.mozPreservesPitch = true
     pitchMedia.webkitPreservesPitch = true
 
-    await this.waitForLoadedMetadata(media)
+    await this.waitForLoadedMetadata(timelineMedia)
     if (this.cancelled) {
       throw new Error('Export cancelled')
     }
@@ -302,9 +318,41 @@ export class AudioProcessor {
     const audioContext = new AudioContext()
     const destinationNode = audioContext.createMediaStreamDestination()
 
-    // Connect original video audio
-    const sourceNode = audioContext.createMediaElementSource(media)
-    sourceNode.connect(destinationNode)
+    let timelineAudioSourceNode: MediaElementAudioSourceNode | null = null
+    if (sourceAudioFallbackPaths.length === 0) {
+      timelineAudioSourceNode = audioContext.createMediaElementSource(timelineMedia)
+      timelineAudioSourceNode.connect(destinationNode)
+    }
+
+    const sourceAudioElements: {
+      media: HTMLAudioElement
+      sourceNode: MediaElementAudioSourceNode
+      cleanup: () => void
+    }[] = []
+
+    for (const sourceAudioPath of sourceAudioFallbackPaths) {
+      const sourceFileSource = await resolveMediaElementSource(sourceAudioPath)
+      const audioEl = document.createElement('audio')
+      audioEl.src = sourceFileSource.src
+      audioEl.preload = 'auto'
+      try {
+        await this.waitForLoadedMetadata(audioEl)
+      } catch {
+        sourceFileSource.revoke()
+        console.warn('[AudioProcessor] Failed to load source audio fallback:', sourceAudioPath)
+        continue
+      }
+      if (this.cancelled) throw new Error('Export cancelled')
+
+      const sourceNode = audioContext.createMediaElementSource(audioEl)
+      sourceNode.connect(destinationNode)
+
+      sourceAudioElements.push({
+        media: audioEl,
+        sourceNode,
+        cleanup: sourceFileSource.revoke,
+      })
+    }
 
     // Prepare external audio region elements
     const audioRegionElements: {
@@ -352,8 +400,8 @@ export class AudioProcessor {
         await audioContext.resume()
       }
 
-      await this.seekTo(media, 0)
-      await media.play()
+      await this.seekTo(timelineMedia, 0)
+      await timelineMedia.play()
 
       await new Promise<void>((resolve, reject) => {
         const cleanup = () => {
@@ -361,8 +409,8 @@ export class AudioProcessor {
             cancelAnimationFrame(rafId)
             rafId = null
           }
-          media.removeEventListener('error', onError)
-          media.removeEventListener('ended', onEnded)
+          timelineMedia.removeEventListener('error', onError)
+          timelineMedia.removeEventListener('ended', onEnded)
         }
 
         const onError = () => {
@@ -382,23 +430,54 @@ export class AudioProcessor {
             return
           }
 
-          const currentTimeMs = media.currentTime * 1000
+          let currentTimeMs = timelineMedia.currentTime * 1000
           const activeTrimRegion = this.findActiveTrimRegion(currentTimeMs, trimRegions)
 
-          if (activeTrimRegion && !media.paused && !media.ended) {
+          if (activeTrimRegion && !timelineMedia.paused && !timelineMedia.ended) {
             const skipToTime = activeTrimRegion.endMs / 1000
-            if (skipToTime >= media.duration) {
-              media.pause()
+            if (skipToTime >= timelineMedia.duration) {
+              timelineMedia.pause()
               cleanup()
               resolve()
               return
             }
-            media.currentTime = skipToTime
-          } else {
-            const activeSpeedRegion = this.findActiveSpeedRegion(currentTimeMs, speedRegions)
-            const playbackRate = activeSpeedRegion ? activeSpeedRegion.speed : 1
-            if (Math.abs(media.playbackRate - playbackRate) > 0.0001) {
-              media.playbackRate = playbackRate
+            timelineMedia.currentTime = skipToTime
+            currentTimeMs = skipToTime * 1000
+          }
+
+          const activeSpeedRegion = this.findActiveSpeedRegion(currentTimeMs, speedRegions)
+          const playbackRate = activeSpeedRegion ? activeSpeedRegion.speed : 1
+          if (Math.abs(timelineMedia.playbackRate - playbackRate) > 0.0001) {
+            timelineMedia.playbackRate = playbackRate
+          }
+
+          for (const entry of sourceAudioElements) {
+            const audioEl = entry.media
+            const targetTimeSec = Math.max(
+              0,
+              Math.min(
+                currentTimeMs / 1000,
+                Number.isFinite(audioEl.duration) ? audioEl.duration : currentTimeMs / 1000,
+              ),
+            )
+
+            if (Math.abs(audioEl.playbackRate - playbackRate) > 0.0001) {
+              audioEl.playbackRate = playbackRate
+            }
+
+            const atEnd = Number.isFinite(audioEl.duration) && targetTimeSec >= audioEl.duration
+            if (atEnd) {
+              if (!audioEl.paused) {
+                audioEl.pause()
+              }
+              continue
+            }
+
+            if (audioEl.paused) {
+              audioEl.currentTime = targetTimeSec
+              audioEl.play().catch(() => {})
+            } else if (Math.abs(audioEl.currentTime - targetTimeSec) > 0.3) {
+              audioEl.currentTime = targetTimeSec
             }
           }
 
@@ -422,7 +501,7 @@ export class AudioProcessor {
             }
           }
 
-          if (!media.paused && !media.ended) {
+          if (!timelineMedia.paused && !timelineMedia.ended) {
             rafId = requestAnimationFrame(tick)
           } else {
             cleanup()
@@ -430,15 +509,26 @@ export class AudioProcessor {
           }
         }
 
-        media.addEventListener('error', onError, { once: true })
-        media.addEventListener('ended', onEnded, { once: true })
+        timelineMedia.addEventListener('error', onError, { once: true })
+        timelineMedia.addEventListener('ended', onEnded, { once: true })
         rafId = requestAnimationFrame(tick)
       })
     } finally {
       if (rafId !== null) {
         cancelAnimationFrame(rafId)
       }
-      media.pause()
+      timelineMedia.pause()
+      timelineAudioSourceNode?.disconnect()
+      timelineMedia.src = ''
+      timelineMedia.load()
+      timelineMediaSource.revoke()
+      for (const entry of sourceAudioElements) {
+        entry.media.pause()
+        entry.sourceNode.disconnect()
+        entry.media.src = ''
+        entry.media.load()
+        entry.cleanup()
+      }
       for (const entry of audioRegionElements) {
         entry.media.pause()
         entry.sourceNode.disconnect()

src/lib/exporter/videoExporter.ts

@@ -56,6 +56,7 @@ interface VideoExporterConfig extends ExportConfig {
 	cursorClickBounceDuration?: number;
 	cursorSway?: number;
 	audioRegions?: AudioRegion[];
+	sourceAudioFallbackPaths?: string[];
 	previewWidth?: number;
 	previewHeight?: number;
 	onProgress?: (progress: ExportProgress) => void;
@@ -139,7 +140,8 @@ export class VideoExporter {
 			await this.initializeEncoder();
 
 			const hasAudioRegions = (this.config.audioRegions ?? []).length > 0;
-			const hasAudio = videoInfo.hasAudio || hasAudioRegions;
+			const hasSourceAudioFallback = (this.config.sourceAudioFallbackPaths ?? []).length > 0;
+			const hasAudio = videoInfo.hasAudio || hasAudioRegions || hasSourceAudioFallback;
 
 			// Initialize muxer
 			this.muxer = new VideoMuxer(this.config, hasAudio);
@@ -196,17 +198,18 @@ export class VideoExporter {
 
 			if (hasAudio && !this.cancelled) {
 				const demuxer = this.streamingDecoder.getDemuxer();
-				if (demuxer || hasAudioRegions) {
+				if (demuxer || hasAudioRegions || hasSourceAudioFallback) {
 					this.audioProcessor = new AudioProcessor();
 					await this.awaitWithWindowsTimeout(
 						this.audioProcessor.process(
-							demuxer!,
+							demuxer,
 							this.muxer!,
 							this.config.videoUrl,
 							this.config.trimRegions,
 							this.config.speedRegions,
 							undefined,
 							this.config.audioRegions,
+							this.config.sourceAudioFallbackPaths,
 						),
 						"audio processing",
 					);