From 745b774df0cee25c07c30f4c53634e53907cd09f Mon Sep 17 00:00:00 2001 From: webadderall <131426131+webadderall@users.noreply.github.com> Date: Tue, 21 Apr 2026 22:00:38 +1000 Subject: [PATCH] Fix embedded audio fallback handling and audio diagnostics --- .../ipc/recording/windowsFallbacks.test.ts | 23 ++++ electron/ipc/recording/windowsFallbacks.ts | 11 ++ electron/ipc/register/recording.ts | 42 ++++--- src/components/video-editor/VideoEditor.tsx | 46 ++++++-- src/lib/exporter/audioEncoder.test.ts | 39 ++++++ src/lib/exporter/audioEncoder.ts | 16 +-- src/lib/exporter/mediaResource.test.ts | 34 ++++++ src/lib/exporter/mediaResource.ts | 111 ++++++++++++++++++ src/lib/exporter/sourceAudioFallback.test.ts | 43 +++++++ src/lib/exporter/sourceAudioFallback.ts | 22 ++++ 10 files changed, 356 insertions(+), 31 deletions(-) create mode 100644 electron/ipc/recording/windowsFallbacks.test.ts create mode 100644 electron/ipc/recording/windowsFallbacks.ts create mode 100644 src/lib/exporter/audioEncoder.test.ts create mode 100644 src/lib/exporter/mediaResource.test.ts create mode 100644 src/lib/exporter/mediaResource.ts create mode 100644 src/lib/exporter/sourceAudioFallback.test.ts create mode 100644 src/lib/exporter/sourceAudioFallback.ts diff --git a/electron/ipc/recording/windowsFallbacks.test.ts b/electron/ipc/recording/windowsFallbacks.test.ts new file mode 100644 index 00000000..9945210e --- /dev/null +++ b/electron/ipc/recording/windowsFallbacks.test.ts @@ -0,0 +1,23 @@ +import { describe, expect, it } from "vitest"; + +import { shouldUseWindowsBrowserMicrophoneFallback } from "./windowsFallbacks"; + +describe("shouldUseWindowsBrowserMicrophoneFallback", () => { + it("returns true when native Windows mic initialization fails", () => { + expect( + shouldUseWindowsBrowserMicrophoneFallback( + "WARNING: Failed to initialize WASAPI mic capture\nRecording started", + { capturesMicrophone: true }, + ), + ).toBe(true); + }); + + it("returns false when microphone capture was not requested", () => { + expect( + shouldUseWindowsBrowserMicrophoneFallback( + "WARNING: Failed to initialize WASAPI mic capture\nRecording started", + { capturesMicrophone: false }, + ), + ).toBe(false); + }); +}); \ No newline at end of file diff --git a/electron/ipc/recording/windowsFallbacks.ts b/electron/ipc/recording/windowsFallbacks.ts new file mode 100644 index 00000000..b8e6e709 --- /dev/null +++ b/electron/ipc/recording/windowsFallbacks.ts @@ -0,0 +1,11 @@ +const WINDOWS_MIC_CAPTURE_INIT_WARNING = "WARNING: Failed to initialize WASAPI mic capture"; + +export function shouldUseWindowsBrowserMicrophoneFallback( + captureOutput: string, + options?: { capturesMicrophone?: boolean }, +) { + return ( + Boolean(options?.capturesMicrophone) && + captureOutput.includes(WINDOWS_MIC_CAPTURE_INIT_WARNING) + ); +} \ No newline at end of file diff --git a/electron/ipc/register/recording.ts b/electron/ipc/register/recording.ts index 4fd5f16a..b4241613 100644 --- a/electron/ipc/register/recording.ts +++ b/electron/ipc/register/recording.ts @@ -94,6 +94,7 @@ import { attachWindowsCaptureLifecycle, muxNativeWindowsVideoWithAudio, } from "../recording/windows"; +import { shouldUseWindowsBrowserMicrophoneFallback } from "../recording/windowsFallbacks"; import { waitForNativeCaptureStart, waitForNativeCaptureStop, @@ -189,6 +190,9 @@ export function registerRecordingHandlers( const recordingsDir = await getRecordingsDir() const timestamp = Date.now() const outputPath = path.join(recordingsDir, `recording-${timestamp}.mp4`) + let captureOutput = '' + let systemAudioPath: string | null = null + let microphonePath: string | null = null const resolvedDisplay = resolveWindowsCaptureDisplay( source, getScreen().getAllDisplays(), @@ -207,20 +211,20 @@ export function registerRecordingHandlers( } if (options?.capturesSystemAudio) { - const audioPath = path.join(recordingsDir, `recording-${timestamp}.system.wav`) + systemAudioPath = path.join(recordingsDir, `recording-${timestamp}.system.wav`) config.captureSystemAudio = true - config.audioOutputPath = audioPath - setWindowsSystemAudioPath(audioPath) + config.audioOutputPath = systemAudioPath + setWindowsSystemAudioPath(systemAudioPath) } if (options?.capturesMicrophone) { - const micPath = path.join(recordingsDir, `recording-${timestamp}.mic.wav`) + microphonePath = path.join(recordingsDir, `recording-${timestamp}.mic.wav`) config.captureMic = true - config.micOutputPath = micPath + config.micOutputPath = microphonePath if (options.microphoneLabel) { config.micDeviceName = options.microphoneLabel } - setWindowsMicAudioPath(micPath) + setWindowsMicAudioPath(microphonePath) } recordNativeCaptureDiagnostics({ @@ -233,8 +237,8 @@ export function registerRecordingHandlers( windowHandle: typeof config.windowHandle === 'number' ? config.windowHandle : null, helperPath: exePath, outputPath, - systemAudioPath: windowsSystemAudioPath, - microphonePath: windowsMicAudioPath, + systemAudioPath, + microphonePath, }) setWindowsCaptureOutputBuffer('') @@ -249,13 +253,23 @@ export function registerRecordingHandlers( attachWindowsCaptureLifecycle(wcProc) wcProc.stdout.on('data', (chunk: Buffer) => { - setWindowsCaptureOutputBuffer(windowsCaptureOutputBuffer + chunk.toString()) + captureOutput += chunk.toString() + setWindowsCaptureOutputBuffer(captureOutput) }) wcProc.stderr.on('data', (chunk: Buffer) => { - setWindowsCaptureOutputBuffer(windowsCaptureOutputBuffer + chunk.toString()) + captureOutput += chunk.toString() + setWindowsCaptureOutputBuffer(captureOutput) }) await waitForWindowsCaptureStart(wcProc) + const microphoneFallbackRequired = shouldUseWindowsBrowserMicrophoneFallback( + captureOutput, + options, + ) + if (microphoneFallbackRequired) { + microphonePath = null + setWindowsMicAudioPath(null) + } setWindowsNativeCaptureActive(true) setNativeScreenRecordingActive(true) recordNativeCaptureDiagnostics({ @@ -268,11 +282,11 @@ export function registerRecordingHandlers( windowHandle: typeof config.windowHandle === 'number' ? config.windowHandle : null, helperPath: exePath, outputPath, - systemAudioPath: windowsSystemAudioPath, - microphonePath: windowsMicAudioPath, - processOutput: windowsCaptureOutputBuffer.trim() || undefined, + systemAudioPath, + microphonePath, + processOutput: captureOutput.trim() || undefined, }) - return { success: true } + return { success: true, microphoneFallbackRequired } } catch (error) { recordNativeCaptureDiagnostics({ backend: 'windows-wgc', diff --git a/src/components/video-editor/VideoEditor.tsx b/src/components/video-editor/VideoEditor.tsx index fc3c3914..d55c64b6 100644 --- a/src/components/video-editor/VideoEditor.tsx +++ b/src/components/video-editor/VideoEditor.tsx @@ -63,6 +63,7 @@ import { VideoExporter, } from "@/lib/exporter"; import { resolveMediaElementSource } from "@/lib/exporter/localMediaSource"; +import { resolveSourceAudioFallbackPaths } from "@/lib/exporter/sourceAudioFallback"; import { clampMediaTimeToDuration, estimateCompanionAudioStartDelaySeconds, @@ -239,6 +240,7 @@ async function writeSmokeExportReport( } const DEFAULT_MP4_EXPORT_FRAME_RATE: ExportMp4FrameRate = 30; +const SOURCE_AUDIO_FALLBACK_TOAST_ID = "source-audio-fallback-error"; function getEncodingModeBitrateMultiplier(encodingMode: ExportEncodingMode): number { switch (encodingMode) { @@ -1203,7 +1205,15 @@ export default function VideoEditor() { () => videoSourcePath ?? (videoPath ? fromFileUrl(videoPath) : null), [videoPath, videoSourcePath], ); - const hasSourceAudioFallback = sourceAudioFallbackPaths.length > 0; + const { + hasEmbeddedSourceAudio, + externalAudioPaths: previewSourceAudioFallbackPaths, + } = useMemo( + () => resolveSourceAudioFallbackPaths(currentSourcePath, sourceAudioFallbackPaths), + [currentSourcePath, sourceAudioFallbackPaths], + ); + const shouldMutePreviewVideo = + !hasEmbeddedSourceAudio && previewSourceAudioFallbackPaths.length > 0; useEffect(() => { let cancelled = false; @@ -1222,10 +1232,26 @@ export default function VideoEditor() { if (cancelled) { return; } - setSourceAudioFallbackPaths(result.success ? (result.paths ?? []) : []); - } catch { + if (!result.success) { + setSourceAudioFallbackPaths([]); + toast.warning( + result.error + ? `Could not load companion audio sources: ${summarizeErrorMessage(result.error)}` + : "Could not load companion audio sources. Playback and export may miss microphone audio.", + { id: SOURCE_AUDIO_FALLBACK_TOAST_ID, duration: 10000 }, + ); + return; + } + + toast.dismiss(SOURCE_AUDIO_FALLBACK_TOAST_ID); + setSourceAudioFallbackPaths(result.paths ?? []); + } catch (error) { if (!cancelled) { setSourceAudioFallbackPaths([]); + toast.warning( + `Could not load companion audio sources: ${summarizeErrorMessage(String(error))}`, + { id: SOURCE_AUDIO_FALLBACK_TOAST_ID, duration: 10000 }, + ); } } })(); @@ -3455,7 +3481,7 @@ export default function VideoEditor() { useEffect(() => { let cancelled = false; const existing = sourceAudioElementsRef.current; - const currentIds = new Set(sourceAudioFallbackPaths); + const currentIds = new Set(previewSourceAudioFallbackPaths); for (const [id, audio] of existing) { if (!currentIds.has(id)) { @@ -3468,7 +3494,7 @@ export default function VideoEditor() { } } - for (const audioPath of sourceAudioFallbackPaths) { + for (const audioPath of previewSourceAudioFallbackPaths) { let audio = existing.get(audioPath); if (!audio) { audio = new Audio(); @@ -3504,14 +3530,14 @@ export default function VideoEditor() { audio.volume = Math.max(0, Math.min(1, previewVolume)); } - if (sourceAudioFallbackPaths.length === 0) { + if (previewSourceAudioFallbackPaths.length === 0) { lastSourceAudioSyncTimeRef.current = null; } return () => { cancelled = true; }; - }, [previewVolume, sourceAudioFallbackPaths]); + }, [previewSourceAudioFallbackPaths, previewVolume]); useEffect(() => { return () => { @@ -3579,7 +3605,7 @@ export default function VideoEditor() { }, [isPlaying, currentTime, audioRegions, speedRegions]); useEffect(() => { - if (sourceAudioFallbackPaths.length === 0) { + if (previewSourceAudioFallbackPaths.length === 0) { lastSourceAudioSyncTimeRef.current = null; return; } @@ -3631,7 +3657,7 @@ export default function VideoEditor() { } lastSourceAudioSyncTimeRef.current = currentTime; - }, [currentTime, duration, isPlaying, sourceAudioFallbackPaths, speedRegions]); + }, [currentTime, duration, isPlaying, previewSourceAudioFallbackPaths, speedRegions]); const showExportSuccessToast = useCallback((filePath: string) => { toast.success(`Exported successfully to ${filePath}`, { @@ -5180,7 +5206,7 @@ export default function VideoEditor() { cursorClickBounceDuration } cursorSway={cursorSway} - volume={hasSourceAudioFallback ? 0 : previewVolume} + volume={shouldMutePreviewVideo ? 0 : previewVolume} /> diff --git a/src/lib/exporter/audioEncoder.test.ts b/src/lib/exporter/audioEncoder.test.ts new file mode 100644 index 00000000..2b67ddaa --- /dev/null +++ b/src/lib/exporter/audioEncoder.test.ts @@ -0,0 +1,39 @@ +import { describe, expect, it, vi } from "vitest"; + +import { AudioProcessor } from "./audioEncoder"; + +describe("AudioProcessor offline render preparation", () => { + it("keeps embedded source audio separate from external companion sidecars", async () => { + const processor = new AudioProcessor(); + const mainBuffer = { duration: 10, numberOfChannels: 2 } as AudioBuffer; + const micBuffer = { duration: 9.5, numberOfChannels: 1 } as AudioBuffer; + + const decodeAudioFromUrl = vi + .spyOn(processor as never, "decodeAudioFromUrl") + .mockImplementation(async (url: string) => { + if (url === "file:///tmp/recording.mp4") { + return mainBuffer; + } + if (url === "/tmp/recording.mic.wav") { + return micBuffer; + } + return null; + }); + vi.spyOn(processor as never, "getMediaDurationSec").mockResolvedValue(10); + + const prepared = await (processor as never).prepareOfflineRender( + "file:///tmp/recording.mp4", + [], + [], + [], + ["/tmp/recording.mp4", "/tmp/recording.mic.wav"], + ); + + expect(prepared.mainBuffer).toBe(mainBuffer); + expect(prepared.companionEntries).toHaveLength(1); + expect(prepared.companionEntries[0]?.buffer).toBe(micBuffer); + expect(decodeAudioFromUrl).toHaveBeenCalledWith("file:///tmp/recording.mp4"); + expect(decodeAudioFromUrl).toHaveBeenCalledWith("/tmp/recording.mic.wav"); + expect(decodeAudioFromUrl).not.toHaveBeenCalledWith("/tmp/recording.mp4"); + }); +}); \ No newline at end of file diff --git a/src/lib/exporter/audioEncoder.ts b/src/lib/exporter/audioEncoder.ts index a5bb79d5..5b8c83bf 100644 --- a/src/lib/exporter/audioEncoder.ts +++ b/src/lib/exporter/audioEncoder.ts @@ -10,6 +10,7 @@ import { } from "@/lib/mediaTiming"; import { resolveMediaElementSource } from "./localMediaSource"; import type { VideoMuxer } from "./muxer"; +import { resolveSourceAudioFallbackPaths } from "./sourceAudioFallback"; const AUDIO_BITRATE = 128_000; const DECODE_BACKPRESSURE_LIMIT = 20; @@ -547,17 +548,18 @@ export class AudioProcessor { if (this.cancelled) throw new Error("Export cancelled"); this.onProgress?.(0); - const hasExternalSources = sourceAudioFallbackPaths.length > 0; + const { externalAudioPaths } = resolveSourceAudioFallbackPaths( + videoUrl, + sourceAudioFallbackPaths, + ); - // Decode primary audio source (streaming decode with bulk fallback) - const mainBuffer = !hasExternalSources - ? await this.decodeAudioFromUrl(videoUrl) - : null; + // Decode embedded source audio separately from companion sidecars. + const mainBuffer = await this.decodeAudioFromUrl(videoUrl); if (this.cancelled) throw new Error("Export cancelled"); // Decode companion / sidecar audio files const companionEntries: Array<{ buffer: AudioBuffer; startDelaySec: number }> = []; - for (const audioPath of sourceAudioFallbackPaths) { + for (const audioPath of externalAudioPaths) { if (this.cancelled) throw new Error("Export cancelled"); const buffer = await this.decodeAudioFromUrl(audioPath); if (!buffer) continue; @@ -593,7 +595,7 @@ export class AudioProcessor { let sourceDurationSec: number; if (mainBuffer) { sourceDurationSec = mainBuffer.duration; - } else if (hasExternalSources || regionEntries.length > 0) { + } else if (externalAudioPaths.length > 0 || regionEntries.length > 0) { sourceDurationSec = await this.getMediaDurationSec(videoUrl); } else { sourceDurationSec = primaryBuffer?.duration ?? 0; diff --git a/src/lib/exporter/mediaResource.test.ts b/src/lib/exporter/mediaResource.test.ts new file mode 100644 index 00000000..48161745 --- /dev/null +++ b/src/lib/exporter/mediaResource.test.ts @@ -0,0 +1,34 @@ +import { describe, expect, it } from "vitest"; + +import { getLocalFilePathFromResource, getResourceFileName } from "./mediaResource"; + +describe("getLocalFilePathFromResource", () => { + it("extracts the path from file URLs", () => { + expect(getLocalFilePathFromResource("file:///tmp/example%20video.mp4")).toBe( + "/tmp/example video.mp4", + ); + }); + + it("extracts the approved file path from loopback media server URLs", () => { + expect( + getLocalFilePathFromResource( + "http://127.0.0.1:4321/video?path=%2Ftmp%2Fexample%20video.mp4", + ), + ).toBe("/tmp/example video.mp4"); + }); + + it("does not treat arbitrary remote URLs as local files", () => { + expect(getLocalFilePathFromResource("https://example.com/video.mp4")).toBeNull(); + }); +}); + +describe("getResourceFileName", () => { + it("uses the source file name for loopback media server URLs", () => { + expect( + getResourceFileName( + "http://127.0.0.1:4321/video?path=%2Ftmp%2Fexample%20video.mp4", + "fallback.mp4", + ), + ).toBe("example video.mp4"); + }); +}); \ No newline at end of file diff --git a/src/lib/exporter/mediaResource.ts b/src/lib/exporter/mediaResource.ts new file mode 100644 index 00000000..83e385eb --- /dev/null +++ b/src/lib/exporter/mediaResource.ts @@ -0,0 +1,111 @@ +const LOOPBACK_MEDIA_SERVER_HOSTS = new Set(["127.0.0.1", "localhost", "::1", "[::1]"]); + +export function isAbsoluteLocalPath(resource: string): boolean { + return ( + resource.startsWith("/") || + /^[A-Za-z]:[\\/]/.test(resource) || + /^\\\\[^\\]+\\[^\\]+/.test(resource) + ); +} + +function fromFileUrl(resource: string): string { + try { + const url = new URL(resource); + const pathname = decodeURIComponent(url.pathname); + + if (url.host && url.host !== "localhost") { + const uncPath = `//${url.host}${pathname.startsWith("/") ? pathname : `/${pathname}`}`; + return uncPath.replace(/\//g, "\\"); + } + + if (/^\/[A-Za-z]:/.test(pathname)) { + return pathname.slice(1); + } + + return pathname; + } catch { + const rawFallbackPath = resource.replace(/^file:\/\//i, ""); + let fallbackPath = rawFallbackPath; + try { + fallbackPath = decodeURIComponent(rawFallbackPath); + } catch { + // Keep raw best-effort path if percent decoding fails. + } + return fallbackPath.replace(/^\/([A-Za-z]:)/, "$1"); + } +} + +function getLoopbackMediaServerPath(resource: string): string | null { + try { + const url = new URL(resource); + if (url.pathname !== "/video") { + return null; + } + + if (!/^https?:$/i.test(url.protocol)) { + return null; + } + + if (!LOOPBACK_MEDIA_SERVER_HOSTS.has(url.hostname.toLowerCase())) { + return null; + } + + const pathParam = url.searchParams.get("path"); + if (!pathParam) { + return null; + } + + if (/^file:\/\//i.test(pathParam)) { + return fromFileUrl(pathParam); + } + + return isAbsoluteLocalPath(pathParam) ? pathParam : null; + } catch { + return null; + } +} + +function getPathBaseName(filePath: string): string { + const normalized = filePath.replace(/\\/g, "/"); + const segments = normalized.split("/").filter(Boolean); + return segments[segments.length - 1] ?? ""; +} + +export function getLocalFilePathFromResource(resource: string): string | null { + if (!resource) { + return null; + } + + if (/^file:\/\//i.test(resource)) { + return fromFileUrl(resource); + } + + const mediaServerPath = getLoopbackMediaServerPath(resource); + if (mediaServerPath) { + return mediaServerPath; + } + + return isAbsoluteLocalPath(resource) ? resource : null; +} + +export function getResourceFileName(resource: string, fallback: string): string { + const localFilePath = getLocalFilePathFromResource(resource); + if (localFilePath) { + const fileName = getPathBaseName(localFilePath); + if (fileName) { + return fileName; + } + } + + try { + const url = new URL(resource); + const fileName = getPathBaseName(decodeURIComponent(url.pathname)); + if (fileName) { + return fileName; + } + } catch { + // Ignore parse errors and fall back to the provided default. + } + + return fallback; +} \ No newline at end of file diff --git a/src/lib/exporter/sourceAudioFallback.test.ts b/src/lib/exporter/sourceAudioFallback.test.ts new file mode 100644 index 00000000..bb0b0a03 --- /dev/null +++ b/src/lib/exporter/sourceAudioFallback.test.ts @@ -0,0 +1,43 @@ +import { describe, expect, it } from "vitest"; + +import { resolveSourceAudioFallbackPaths } from "./sourceAudioFallback"; + +describe("resolveSourceAudioFallbackPaths", () => { + it("treats the video file path as embedded source audio when present in the fallback list", () => { + const videoPath = "/tmp/recording.mp4"; + + expect( + resolveSourceAudioFallbackPaths(videoPath, [videoPath, "/tmp/recording.mic.wav"]), + ).toEqual({ + hasEmbeddedSourceAudio: true, + externalAudioPaths: ["/tmp/recording.mic.wav"], + }); + }); + + it("keeps all fallback paths external when the video has no embedded source audio", () => { + expect( + resolveSourceAudioFallbackPaths("/tmp/recording.mp4", [ + "/tmp/recording.system.wav", + "/tmp/recording.mic.wav", + ]), + ).toEqual({ + hasEmbeddedSourceAudio: false, + externalAudioPaths: [ + "/tmp/recording.system.wav", + "/tmp/recording.mic.wav", + ], + }); + }); + + it("matches embedded source audio when the video resource is a file URL", () => { + expect( + resolveSourceAudioFallbackPaths("file:///tmp/recording.mp4", [ + "/tmp/recording.mp4", + "/tmp/recording.mic.wav", + ]), + ).toEqual({ + hasEmbeddedSourceAudio: true, + externalAudioPaths: ["/tmp/recording.mic.wav"], + }); + }); +}); \ No newline at end of file diff --git a/src/lib/exporter/sourceAudioFallback.ts b/src/lib/exporter/sourceAudioFallback.ts new file mode 100644 index 00000000..0b937614 --- /dev/null +++ b/src/lib/exporter/sourceAudioFallback.ts @@ -0,0 +1,22 @@ +import { getLocalFilePathFromResource } from "./mediaResource"; + +export function resolveSourceAudioFallbackPaths( + videoResource: string | null | undefined, + sourceAudioFallbackPaths: string[] | null | undefined, +) { + const normalizedPaths = (sourceAudioFallbackPaths ?? []).filter( + (audioPath) => typeof audioPath === "string" && audioPath.trim().length > 0, + ); + const localVideoSourcePath = videoResource + ? getLocalFilePathFromResource(videoResource) + : null; + const hasEmbeddedSourceAudio = + Boolean(localVideoSourcePath) && normalizedPaths.includes(localVideoSourcePath); + + return { + hasEmbeddedSourceAudio, + externalAudioPaths: hasEmbeddedSourceAudio + ? normalizedPaths.filter((audioPath) => audioPath !== localVideoSourcePath) + : normalizedPaths, + }; +} \ No newline at end of file